From bb1941445d6adfae6278e8647dfdde3f85364180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Sun, 19 Mar 2023 10:04:06 +0100 Subject: [PATCH 01/96] improve frontend, no reloading. types in views_api, formatting --- __init__.py | 6 ++-- crud.py | 3 +- models.py | 8 ++--- nostr/bech32.py | 33 +++++++++++++------ nostr/client/cbc.py | 12 +++---- nostr/client/client.py | 19 +++++------ nostr/delegation.py | 8 ++--- nostr/event.py | 7 +++-- nostr/key.py | 13 ++++---- nostr/message_pool.py | 3 +- nostr/message_type.py | 9 ++++-- nostr/pow.py | 11 +++++-- nostr/relay.py | 2 ++ nostr/subscription.py | 8 ++--- services.py | 16 +++++----- tasks.py | 4 +-- templates/nostrclient/index.html | 23 ++++++-------- views.py | 14 ++++----- views_api.py | 54 +++++++++++++++++--------------- 19 files changed, 134 insertions(+), 119 deletions(-) diff --git a/__init__.py b/__init__.py index 1555eb9..ace3254 100644 --- a/__init__.py +++ b/__init__.py @@ -1,9 +1,8 @@ from fastapi import APIRouter -from starlette.staticfiles import StaticFiles - from lnbits.db import Database from lnbits.helpers import template_renderer from lnbits.tasks import catch_everything_and_restart +from starlette.staticfiles import StaticFiles db = Database("ext_nostrclient") @@ -22,11 +21,10 @@ def nostr_renderer(): return template_renderer(["lnbits/extensions/nostrclient/templates"]) +from .tasks import init_relays, subscribe_events from .views import * # noqa from .views_api import * # noqa -from .tasks import init_relays, subscribe_events - def nostrclient_start(): loop = asyncio.get_event_loop() diff --git a/crud.py b/crud.py index b1916b7..497fcd7 100644 --- a/crud.py +++ b/crud.py @@ -1,7 +1,8 @@ from typing import List, Optional, Union -from lnbits.helpers import urlsafe_short_hash import shortuuid +from lnbits.helpers import urlsafe_short_hash + from . import db from .models import Relay, RelayList diff --git a/models.py b/models.py index bfbc424..75a086d 100644 --- a/models.py +++ b/models.py @@ -1,12 +1,10 @@ -from typing import List, Dict -from typing import Optional +from dataclasses import dataclass +from typing import Dict, List, Optional from fastapi import Request -from pydantic import BaseModel, Field - from fastapi.param_functions import Query -from dataclasses import dataclass from lnbits.helpers import urlsafe_short_hash +from pydantic import BaseModel, Field class Relay(BaseModel): diff --git a/nostr/bech32.py b/nostr/bech32.py index b068de7..0ae6c80 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -23,21 +23,25 @@ from enum import Enum + class Encoding(Enum): """Enumeration type to list the various supported encodings.""" + BECH32 = 1 BECH32M = 2 + CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" -BECH32M_CONST = 0x2bc830a3 +BECH32M_CONST = 0x2BC830A3 + def bech32_polymod(values): """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] + generator = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] chk = 1 for value in values: top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value + chk = (chk & 0x1FFFFFF) << 5 ^ value for i in range(5): chk ^= generator[i] if ((top >> i) & 1) else 0 return chk @@ -57,6 +61,7 @@ def bech32_verify_checksum(hrp, data): return Encoding.BECH32M return None + def bech32_create_checksum(hrp, data, spec): """Compute the checksum values given HRP and data.""" values = bech32_hrp_expand(hrp) + data @@ -68,26 +73,29 @@ def bech32_create_checksum(hrp, data, spec): def bech32_encode(hrp, data, spec): """Compute a Bech32 string given HRP and data values.""" combined = data + bech32_create_checksum(hrp, data, spec) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) + return hrp + "1" + "".join([CHARSET[d] for d in combined]) + def bech32_decode(bech): """Validate a Bech32/Bech32m string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): + if (any(ord(x) < 33 or ord(x) > 126 for x in bech)) or ( + bech.lower() != bech and bech.upper() != bech + ): return (None, None, None) bech = bech.lower() - pos = bech.rfind('1') + pos = bech.rfind("1") if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: return (None, None, None) - if not all(x in CHARSET for x in bech[pos+1:]): + if not all(x in CHARSET for x in bech[pos + 1 :]): return (None, None, None) hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] + data = [CHARSET.find(x) for x in bech[pos + 1 :]] spec = bech32_verify_checksum(hrp, data) if spec is None: return (None, None, None) return (hrp, data[:-6], spec) + def convertbits(data, frombits, tobits, pad=True): """General power-of-2 base conversion.""" acc = 0 @@ -123,7 +131,12 @@ def decode(hrp, addr): return (None, None) if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: return (None, None) - if data[0] == 0 and spec != Encoding.BECH32 or data[0] != 0 and spec != Encoding.BECH32M: + if ( + data[0] == 0 + and spec != Encoding.BECH32 + or data[0] != 0 + and spec != Encoding.BECH32M + ): return (None, None) return (data[0], decoded) diff --git a/nostr/client/cbc.py b/nostr/client/cbc.py index a41dbc0..e69e8b5 100644 --- a/nostr/client/cbc.py +++ b/nostr/client/cbc.py @@ -1,4 +1,3 @@ - from Cryptodome import Random from Cryptodome.Cipher import AES @@ -11,10 +10,10 @@ key = bytes.fromhex("3aa925cb69eb613e2928f8a18279c78b1dca04541dfd064df2eda66b598 BLOCK_SIZE = 16 -class AESCipher(object): - """This class is compatible with crypto.createCipheriv('aes-256-cbc') - """ +class AESCipher(object): + """This class is compatible with crypto.createCipheriv('aes-256-cbc')""" + def __init__(self, key=None): self.key = key @@ -33,9 +32,10 @@ class AESCipher(object): def decrypt(self, iv, enc_text): cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) return self.unpad(cipher.decrypt(enc_text).decode("UTF-8")) - + + if __name__ == "__main__": aes = AESCipher(key=key) iv, enc_text = aes.encrypt(plain_text) dec_text = aes.decrypt(iv, enc_text) - print(dec_text) \ No newline at end of file + print(dec_text) diff --git a/nostr/client/client.py b/nostr/client/client.py index 6fb885f..5be6089 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,20 +1,15 @@ -from typing import * -import ssl -import time +import base64 import json import os -import base64 - -from ..event import Event -from ..relay_manager import RelayManager -from ..message_type import ClientMessageType -from ..key import PrivateKey, PublicKey +import ssl +import time +from typing import * +from ..event import EncryptedDirectMessage, Event, EventKind from ..filter import Filter, Filters -from ..event import Event, EventKind, EncryptedDirectMessage -from ..relay_manager import RelayManager +from ..key import PrivateKey, PublicKey from ..message_type import ClientMessageType - +from ..relay_manager import RelayManager # from aes import AESCipher from . import cbc diff --git a/nostr/delegation.py b/nostr/delegation.py index 94801f5..8b1c311 100644 --- a/nostr/delegation.py +++ b/nostr/delegation.py @@ -7,23 +7,23 @@ class Delegation: delegator_pubkey: str delegatee_pubkey: str event_kind: int - duration_secs: int = 30*24*60 # default to 30 days + duration_secs: int = 30 * 24 * 60 # default to 30 days signature: str = None # set in PrivateKey.sign_delegation @property def expires(self) -> int: return int(time.time()) + self.duration_secs - + @property def conditions(self) -> str: return f"kind={self.event_kind}&created_at<{self.expires}" - + @property def delegation_token(self) -> str: return f"nostr:delegation:{self.delegatee_pubkey}:{self.conditions}" def get_tag(self) -> list[str]: - """ Called by Event """ + """Called by Event""" return [ "delegation", self.delegator_pubkey, diff --git a/nostr/event.py b/nostr/event.py index b903e0e..65b187d 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -1,10 +1,11 @@ -import time import json +import time from dataclasses import dataclass, field from enum import IntEnum -from typing import List -from secp256k1 import PublicKey from hashlib import sha256 +from typing import List + +from secp256k1 import PublicKey from .message_type import ClientMessageType diff --git a/nostr/key.py b/nostr/key.py index d34697f..8089e11 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,14 +1,15 @@ -import secrets import base64 -import secp256k1 -from cffi import FFI -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.primitives import padding +import secrets from hashlib import sha256 +import secp256k1 +from cffi import FFI +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +from . import bech32 from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind -from . import bech32 class PublicKey: diff --git a/nostr/message_pool.py b/nostr/message_pool.py index d364cf2..373d9fc 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -1,8 +1,9 @@ import json from queue import Queue from threading import Lock -from .message_type import RelayMessageType + from .event import Event +from .message_type import RelayMessageType class EventMessage: diff --git a/nostr/message_type.py b/nostr/message_type.py index 3f5206b..9f3be78 100644 --- a/nostr/message_type.py +++ b/nostr/message_type.py @@ -3,6 +3,7 @@ class ClientMessageType: REQUEST = "REQ" CLOSE = "CLOSE" + class RelayMessageType: EVENT = "EVENT" NOTICE = "NOTICE" @@ -10,6 +11,10 @@ class RelayMessageType: @staticmethod def is_valid(type: str) -> bool: - if type == RelayMessageType.EVENT or type == RelayMessageType.NOTICE or type == RelayMessageType.END_OF_STORED_EVENTS: + if ( + type == RelayMessageType.EVENT + or type == RelayMessageType.NOTICE + or type == RelayMessageType.END_OF_STORED_EVENTS + ): return True - return False \ No newline at end of file + return False diff --git a/nostr/pow.py b/nostr/pow.py index e006288..034ad9a 100644 --- a/nostr/pow.py +++ b/nostr/pow.py @@ -1,7 +1,9 @@ import time + from .event import Event from .key import PrivateKey + def zero_bits(b: int) -> int: n = 0 @@ -14,10 +16,11 @@ def zero_bits(b: int) -> int: return 7 - n + def count_leading_zero_bits(hex_str: str) -> int: total = 0 for i in range(0, len(hex_str) - 2, 2): - bits = zero_bits(int(hex_str[i:i+2], 16)) + bits = zero_bits(int(hex_str[i : i + 2], 16)) total += bits if bits != 8: @@ -25,7 +28,10 @@ def count_leading_zero_bits(hex_str: str) -> int: return total -def mine_event(content: str, difficulty: int, public_key: str, kind: int, tags: list=[]) -> Event: + +def mine_event( + content: str, difficulty: int, public_key: str, kind: int, tags: list = [] +) -> Event: all_tags = [["nonce", "1", str(difficulty)]] all_tags.extend(tags) @@ -43,6 +49,7 @@ def mine_event(content: str, difficulty: int, public_key: str, kind: int, tags: return Event(public_key, content, created_at, kind, all_tags, event_id) + def mine_key(difficulty: int) -> PrivateKey: sk = PrivateKey() num_leading_zero_bits = count_leading_zero_bits(sk.public_key.hex()) diff --git a/nostr/relay.py b/nostr/relay.py index ee78baa..0851c02 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,7 +2,9 @@ import json import time from queue import Queue from threading import Lock + from websocket import WebSocketApp + from .event import Event from .filter import Filters from .message_pool import MessagePool diff --git a/nostr/subscription.py b/nostr/subscription.py index 7afba20..10b5363 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,12 +1,10 @@ from .filter import Filters + class Subscription: - def __init__(self, id: str, filters: Filters=None) -> None: + def __init__(self, id: str, filters: Filters = None) -> None: self.id = id self.filters = filters def to_json_object(self): - return { - "id": self.id, - "filters": self.filters.to_json_array() - } + return {"id": self.id, "filters": self.filters.to_json_array()} diff --git a/services.py b/services.py index fa548bf..243af58 100644 --- a/services.py +++ b/services.py @@ -1,19 +1,17 @@ import asyncio import json from typing import List, Union -from .models import RelayList, Relay, Event, Filter, Filters +from fastapi import WebSocket, WebSocketDisconnect +from lnbits.helpers import urlsafe_short_hash + +from .models import Event, Filter, Filters, Relay, RelayList from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters -from .tasks import ( - client, - received_event_queue, - received_subscription_events, - received_subscription_eosenotices, -) -from fastapi import WebSocket, WebSocketDisconnect -from lnbits.helpers import urlsafe_short_hash +from .tasks import (client, received_event_queue, + received_subscription_eosenotices, + received_subscription_events) class NostrRouter: diff --git a/tasks.py b/tasks.py index 40d8aec..8bbf8aa 100644 --- a/tasks.py +++ b/tasks.py @@ -4,11 +4,11 @@ import threading from .nostr.client.client import NostrClient from .nostr.event import Event -from .nostr.message_pool import EventMessage, NoticeMessage, EndOfStoredEventsMessage from .nostr.key import PublicKey +from .nostr.message_pool import (EndOfStoredEventsMessage, EventMessage, + NoticeMessage) from .nostr.relay_manager import RelayManager - client = NostrClient( connect=False, ) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 8ddd362..2a6cf60 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -75,7 +75,7 @@ - +
Your endpoint: @@ -101,11 +101,11 @@
-
{{SITE_TITLE}} Nostrclient Extension
+
Nostrclient Extension

This extension is a always-on nostr client that other extensions can - use to send and receive events on nostr. - + use to send and receive events on nostr. + Add multiple nostr relays to connect to. The extension then opens a websocket for you to use at

@@ -197,10 +197,8 @@ ) .then(function (response) { if (response.data) { - console.log(response.data) response.data.map(maplrelays) self.nostrrelayLinks = response.data - console.log(self.nostrrelayLinks) } }) .catch(function (error) { @@ -219,10 +217,10 @@ message: `Invalid relay URL.`, caption: "Should start with 'wss://'' or 'ws://'" }) - return + return false; } console.log('ADD RELAY ' + this.relayToAdd) - var self = this + let that = this LNbits.api .request( 'POST', @@ -231,17 +229,17 @@ {url: this.relayToAdd} ) .then(function (response) { + console.log("response:", response) if (response.data) { - console.log(response.data) response.data.map(maplrelays) - self.nostrrelayLinks = response.data - console.log(self.nostrrelayLinks) + that.nostrrelayLinks = response.data + that.relayToAdd = '' } }) .catch(function (error) { LNbits.utils.notifyApiError(error) }) - location.reload() + return false; }, deleteRelay(url) { console.log('DELETE RELAY ' + url) @@ -260,7 +258,6 @@ .catch(function (error) { LNbits.utils.notifyApiError(error) }) - location.reload() }, exportlnurldeviceCSV: function () { var self = this diff --git a/views.py b/views.py index 90dd31c..7f07f5d 100644 --- a/views.py +++ b/views.py @@ -1,20 +1,18 @@ -from http import HTTPStatus import asyncio +from http import HTTPStatus + +# FastAPI good for incoming from fastapi import Request from fastapi.param_functions import Query from fastapi.params import Depends from fastapi.templating import Jinja2Templates -from starlette.exceptions import HTTPException -from starlette.responses import HTMLResponse -from . import nostrclient_ext, nostr_renderer - -# FastAPI good for incoming -from fastapi import Request from lnbits.core.crud import update_payment_status from lnbits.core.models import User from lnbits.core.views.api import api_payment -from lnbits.decorators import check_user_exists, check_admin +from lnbits.decorators import check_admin, check_user_exists +from starlette.responses import HTMLResponse +from . import nostr_renderer, nostrclient_ext templates = Jinja2Templates(directory="templates") diff --git a/views_api.py b/views_api.py index d2c74fe..1cf7a19 100644 --- a/views_api.py +++ b/views_api.py @@ -1,33 +1,25 @@ -from http import HTTPStatus import asyncio -from fastapi import WebSocket -from fastapi.params import Depends +from http import HTTPStatus +from typing import Optional + +from fastapi import Depends, WebSocket +from lnbits.decorators import check_admin +from lnbits.helpers import urlsafe_short_hash +from loguru import logger +from starlette.exceptions import HTTPException from . import nostrclient_ext -from .tasks import client -from loguru import logger - -from .crud import get_relays, add_relay, delete_relay -from .models import RelayList, Relay - +from .crud import add_relay, delete_relay, get_relays +from .models import Relay, RelayList from .services import NostrRouter - -from lnbits.decorators import ( - WalletTypeInfo, - get_key_type, - require_admin_key, - check_admin, -) - -from lnbits.helpers import urlsafe_short_hash -from .tasks import init_relays +from .tasks import client, init_relays # we keep this in all_routers: list[NostrRouter] = [] @nostrclient_ext.get("/api/v1/relays") -async def api_get_relays(): # type: ignore +async def api_get_relays() -> RelayList: relays = RelayList(__root__=[]) for url, r in client.relay_manager.relays.items(): status_text = ( @@ -52,20 +44,30 @@ async def api_get_relays(): # type: ignore @nostrclient_ext.post( "/api/v1/relay", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] ) -async def api_add_relay(relay: Relay): # type: ignore - assert relay.url, "no URL" +async def api_add_relay(relay: Relay) -> Optional[RelayList]: + if not relay.url: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, detail=f"Relay url not provided." + ) if relay.url in client.relay_manager.relays: - return + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail=f"Relay: {relay.url} already exists.", + ) relay.id = urlsafe_short_hash() await add_relay(relay) await init_relays() + return await get_relays() @nostrclient_ext.delete( "/api/v1/relay", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] ) -async def api_delete_relay(relay: Relay): # type: ignore - assert relay.url +async def api_delete_relay(relay: Relay) -> None: + if not relay.url: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, detail=f"Relay url not provided." + ) client.relay_manager.remove_relay(relay.url) await delete_relay(relay) @@ -91,7 +93,7 @@ async def api_stop(): @nostrclient_ext.websocket("/api/v1/relay") -async def ws_relay(websocket: WebSocket): +async def ws_relay(websocket: WebSocket) -> None: """Relay multiplexer: one client (per endpoint) <-> multiple relays""" await websocket.accept() router = NostrRouter(websocket) From 4549d0af092846974ea615a3396c1a106b196a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Sun, 19 Mar 2023 11:15:26 +0100 Subject: [PATCH 02/96] revert formatting nostr --- nostr/bech32.py | 33 ++++++++++----------------------- nostr/client/cbc.py | 10 +++++----- nostr/client/client.py | 21 +++++++++++++-------- nostr/delegation.py | 8 ++++---- nostr/event.py | 5 ++--- nostr/key.py | 9 ++++----- nostr/message_pool.py | 3 +-- nostr/message_type.py | 9 ++------- nostr/pow.py | 11 ++--------- nostr/relay.py | 2 -- nostr/subscription.py | 8 +++++--- 11 files changed, 48 insertions(+), 71 deletions(-) diff --git a/nostr/bech32.py b/nostr/bech32.py index 0ae6c80..b068de7 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -23,25 +23,21 @@ from enum import Enum - class Encoding(Enum): """Enumeration type to list the various supported encodings.""" - BECH32 = 1 BECH32M = 2 - CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" -BECH32M_CONST = 0x2BC830A3 - +BECH32M_CONST = 0x2bc830a3 def bech32_polymod(values): """Internal function that computes the Bech32 checksum.""" - generator = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] + generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] chk = 1 for value in values: top = chk >> 25 - chk = (chk & 0x1FFFFFF) << 5 ^ value + chk = (chk & 0x1ffffff) << 5 ^ value for i in range(5): chk ^= generator[i] if ((top >> i) & 1) else 0 return chk @@ -61,7 +57,6 @@ def bech32_verify_checksum(hrp, data): return Encoding.BECH32M return None - def bech32_create_checksum(hrp, data, spec): """Compute the checksum values given HRP and data.""" values = bech32_hrp_expand(hrp) + data @@ -73,29 +68,26 @@ def bech32_create_checksum(hrp, data, spec): def bech32_encode(hrp, data, spec): """Compute a Bech32 string given HRP and data values.""" combined = data + bech32_create_checksum(hrp, data, spec) - return hrp + "1" + "".join([CHARSET[d] for d in combined]) - + return hrp + '1' + ''.join([CHARSET[d] for d in combined]) def bech32_decode(bech): """Validate a Bech32/Bech32m string, and determine HRP and data.""" - if (any(ord(x) < 33 or ord(x) > 126 for x in bech)) or ( - bech.lower() != bech and bech.upper() != bech - ): + if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or + (bech.lower() != bech and bech.upper() != bech)): return (None, None, None) bech = bech.lower() - pos = bech.rfind("1") + pos = bech.rfind('1') if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: return (None, None, None) - if not all(x in CHARSET for x in bech[pos + 1 :]): + if not all(x in CHARSET for x in bech[pos+1:]): return (None, None, None) hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos + 1 :]] + data = [CHARSET.find(x) for x in bech[pos+1:]] spec = bech32_verify_checksum(hrp, data) if spec is None: return (None, None, None) return (hrp, data[:-6], spec) - def convertbits(data, frombits, tobits, pad=True): """General power-of-2 base conversion.""" acc = 0 @@ -131,12 +123,7 @@ def decode(hrp, addr): return (None, None) if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: return (None, None) - if ( - data[0] == 0 - and spec != Encoding.BECH32 - or data[0] != 0 - and spec != Encoding.BECH32M - ): + if data[0] == 0 and spec != Encoding.BECH32 or data[0] != 0 and spec != Encoding.BECH32M: return (None, None) return (data[0], decoded) diff --git a/nostr/client/cbc.py b/nostr/client/cbc.py index e69e8b5..a41dbc0 100644 --- a/nostr/client/cbc.py +++ b/nostr/client/cbc.py @@ -1,3 +1,4 @@ + from Cryptodome import Random from Cryptodome.Cipher import AES @@ -10,10 +11,10 @@ key = bytes.fromhex("3aa925cb69eb613e2928f8a18279c78b1dca04541dfd064df2eda66b598 BLOCK_SIZE = 16 - class AESCipher(object): - """This class is compatible with crypto.createCipheriv('aes-256-cbc')""" + """This class is compatible with crypto.createCipheriv('aes-256-cbc') + """ def __init__(self, key=None): self.key = key @@ -32,10 +33,9 @@ class AESCipher(object): def decrypt(self, iv, enc_text): cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) return self.unpad(cipher.decrypt(enc_text).decode("UTF-8")) - - + if __name__ == "__main__": aes = AESCipher(key=key) iv, enc_text = aes.encrypt(plain_text) dec_text = aes.decrypt(iv, enc_text) - print(dec_text) + print(dec_text) \ No newline at end of file diff --git a/nostr/client/client.py b/nostr/client/client.py index 5be6089..6fb885f 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,15 +1,20 @@ -import base64 -import json -import os +from typing import * import ssl import time -from typing import * +import json +import os +import base64 -from ..event import EncryptedDirectMessage, Event, EventKind -from ..filter import Filter, Filters -from ..key import PrivateKey, PublicKey -from ..message_type import ClientMessageType +from ..event import Event from ..relay_manager import RelayManager +from ..message_type import ClientMessageType +from ..key import PrivateKey, PublicKey + +from ..filter import Filter, Filters +from ..event import Event, EventKind, EncryptedDirectMessage +from ..relay_manager import RelayManager +from ..message_type import ClientMessageType + # from aes import AESCipher from . import cbc diff --git a/nostr/delegation.py b/nostr/delegation.py index 8b1c311..94801f5 100644 --- a/nostr/delegation.py +++ b/nostr/delegation.py @@ -7,23 +7,23 @@ class Delegation: delegator_pubkey: str delegatee_pubkey: str event_kind: int - duration_secs: int = 30 * 24 * 60 # default to 30 days + duration_secs: int = 30*24*60 # default to 30 days signature: str = None # set in PrivateKey.sign_delegation @property def expires(self) -> int: return int(time.time()) + self.duration_secs - + @property def conditions(self) -> str: return f"kind={self.event_kind}&created_at<{self.expires}" - + @property def delegation_token(self) -> str: return f"nostr:delegation:{self.delegatee_pubkey}:{self.conditions}" def get_tag(self) -> list[str]: - """Called by Event""" + """ Called by Event """ return [ "delegation", self.delegator_pubkey, diff --git a/nostr/event.py b/nostr/event.py index 65b187d..b903e0e 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -1,11 +1,10 @@ -import json import time +import json from dataclasses import dataclass, field from enum import IntEnum -from hashlib import sha256 from typing import List - from secp256k1 import PublicKey +from hashlib import sha256 from .message_type import ClientMessageType diff --git a/nostr/key.py b/nostr/key.py index 8089e11..d34697f 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,15 +1,14 @@ -import base64 import secrets -from hashlib import sha256 - +import base64 import secp256k1 from cffi import FFI -from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives import padding +from hashlib import sha256 -from . import bech32 from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind +from . import bech32 class PublicKey: diff --git a/nostr/message_pool.py b/nostr/message_pool.py index 373d9fc..d364cf2 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -1,9 +1,8 @@ import json from queue import Queue from threading import Lock - -from .event import Event from .message_type import RelayMessageType +from .event import Event class EventMessage: diff --git a/nostr/message_type.py b/nostr/message_type.py index 9f3be78..3f5206b 100644 --- a/nostr/message_type.py +++ b/nostr/message_type.py @@ -3,7 +3,6 @@ class ClientMessageType: REQUEST = "REQ" CLOSE = "CLOSE" - class RelayMessageType: EVENT = "EVENT" NOTICE = "NOTICE" @@ -11,10 +10,6 @@ class RelayMessageType: @staticmethod def is_valid(type: str) -> bool: - if ( - type == RelayMessageType.EVENT - or type == RelayMessageType.NOTICE - or type == RelayMessageType.END_OF_STORED_EVENTS - ): + if type == RelayMessageType.EVENT or type == RelayMessageType.NOTICE or type == RelayMessageType.END_OF_STORED_EVENTS: return True - return False + return False \ No newline at end of file diff --git a/nostr/pow.py b/nostr/pow.py index 034ad9a..e006288 100644 --- a/nostr/pow.py +++ b/nostr/pow.py @@ -1,9 +1,7 @@ import time - from .event import Event from .key import PrivateKey - def zero_bits(b: int) -> int: n = 0 @@ -16,11 +14,10 @@ def zero_bits(b: int) -> int: return 7 - n - def count_leading_zero_bits(hex_str: str) -> int: total = 0 for i in range(0, len(hex_str) - 2, 2): - bits = zero_bits(int(hex_str[i : i + 2], 16)) + bits = zero_bits(int(hex_str[i:i+2], 16)) total += bits if bits != 8: @@ -28,10 +25,7 @@ def count_leading_zero_bits(hex_str: str) -> int: return total - -def mine_event( - content: str, difficulty: int, public_key: str, kind: int, tags: list = [] -) -> Event: +def mine_event(content: str, difficulty: int, public_key: str, kind: int, tags: list=[]) -> Event: all_tags = [["nonce", "1", str(difficulty)]] all_tags.extend(tags) @@ -49,7 +43,6 @@ def mine_event( return Event(public_key, content, created_at, kind, all_tags, event_id) - def mine_key(difficulty: int) -> PrivateKey: sk = PrivateKey() num_leading_zero_bits = count_leading_zero_bits(sk.public_key.hex()) diff --git a/nostr/relay.py b/nostr/relay.py index 0851c02..ee78baa 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,9 +2,7 @@ import json import time from queue import Queue from threading import Lock - from websocket import WebSocketApp - from .event import Event from .filter import Filters from .message_pool import MessagePool diff --git a/nostr/subscription.py b/nostr/subscription.py index 10b5363..7afba20 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,10 +1,12 @@ from .filter import Filters - class Subscription: - def __init__(self, id: str, filters: Filters = None) -> None: + def __init__(self, id: str, filters: Filters=None) -> None: self.id = id self.filters = filters def to_json_object(self): - return {"id": self.id, "filters": self.filters.to_json_array()} + return { + "id": self.id, + "filters": self.filters.to_json_array() + } From d67133ae61ef08d1f27c2987d56423312444cb32 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:17:04 +0100 Subject: [PATCH 03/96] refactor nostrclient --- __init__.py | 5 +--- services.py | 24 +++++++++++++---- tasks.py | 27 +++++++++---------- views.py | 74 ---------------------------------------------------- views_api.py | 18 +++++++------ 5 files changed, 43 insertions(+), 105 deletions(-) diff --git a/__init__.py b/__init__.py index ace3254..29f5658 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +import asyncio from fastapi import APIRouter from lnbits.db import Database from lnbits.helpers import template_renderer @@ -22,13 +23,9 @@ def nostr_renderer(): from .tasks import init_relays, subscribe_events -from .views import * # noqa -from .views_api import * # noqa def nostrclient_start(): loop = asyncio.get_event_loop() loop.create_task(catch_everything_and_restart(init_relays)) - # loop.create_task(catch_everything_and_restart(send_data)) - # loop.create_task(catch_everything_and_restart(receive_data)) loop.create_task(catch_everything_and_restart(subscribe_events)) diff --git a/services.py b/services.py index 243af58..09e9235 100644 --- a/services.py +++ b/services.py @@ -4,14 +4,26 @@ from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect from lnbits.helpers import urlsafe_short_hash +from .nostr.client.client import NostrClient as NostrClientLib from .models import Event, Filter, Filters, Relay, RelayList from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters -from .tasks import (client, received_event_queue, - received_subscription_eosenotices, - received_subscription_events) +from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage + + +received_subscription_events: dict[str, list[Event]] = {} +received_subscription_notices: dict[str, list[NoticeMessage]] = {} +received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} + + +class NostrClient: + def __init__(self): + self.client: NostrClientLib = NostrClientLib(connect=False) + + +nostr = NostrClient() class NostrRouter: @@ -44,7 +56,7 @@ class NostrRouter: json_str = json_str_rewritten # publish data - client.relay_manager.publish_message(json_str) + nostr.client.relay_manager.publish_message(json_str) async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -126,7 +138,9 @@ class NostrRouter: ) fltr = json_data[2] filters = self._marshall_nostr_filters(fltr) - client.relay_manager.add_subscription(subscription_id_rewritten, filters) + nostr.client.relay_manager.add_subscription( + subscription_id_rewritten, filters + ) request_rewritten = json.dumps(["REQ", subscription_id_rewritten, fltr]) return subscription_id_rewritten, request_rewritten return None, None diff --git a/tasks.py b/tasks.py index 8bbf8aa..7894e40 100644 --- a/tasks.py +++ b/tasks.py @@ -2,34 +2,33 @@ import asyncio import ssl import threading -from .nostr.client.client import NostrClient from .nostr.event import Event from .nostr.key import PublicKey -from .nostr.message_pool import (EndOfStoredEventsMessage, EventMessage, - NoticeMessage) +from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage from .nostr.relay_manager import RelayManager - -client = NostrClient( - connect=False, +from .services import ( + nostr, + received_subscription_events, + received_subscription_eosenotices, ) -received_event_queue: asyncio.Queue[EventMessage] = asyncio.Queue(0) -received_subscription_events: dict[str, list[Event]] = {} -received_subscription_notices: dict[str, list[NoticeMessage]] = {} -received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} from .crud import get_relays async def init_relays(): + # reinitialize the entire client + nostr.__init__() + # get relays from db relays = await get_relays() - client.relays = list(set([r.url for r in relays.__root__ if r.url])) - client.connect() + # set relays and connect to them + nostr.client.relays = list(set([r.url for r in relays.__root__ if r.url])) + nostr.client.connect() return async def subscribe_events(): - while not any([r.connected for r in client.relay_manager.relays.values()]): + while not any([r.connected for r in nostr.client.relay_manager.relays.values()]): await asyncio.sleep(2) def callback_events(eventMessage: EventMessage): @@ -65,7 +64,7 @@ async def subscribe_events(): return t = threading.Thread( - target=client.subscribe, + target=nostr.client.subscribe, args=( callback_events, callback_notices, diff --git a/views.py b/views.py index 7f07f5d..ce30b3b 100644 --- a/views.py +++ b/views.py @@ -22,77 +22,3 @@ async def index(request: Request, user: User = Depends(check_admin)): return nostr_renderer().TemplateResponse( "nostrclient/index.html", {"request": request, "user": user.dict()} ) - - -# ##################################################################### -# #################### NOSTR WEBSOCKET THREAD ######################### -# ##### THE QUEUE LOOP THREAD THING THAT LISTENS TO BUNCH OF ########## -# ### WEBSOCKET CONNECTIONS, STORING DATA IN DB/PUSHING TO FRONTEND ### -# ################### VIA updater() FUNCTION ########################## -# ##################################################################### - -# websocket_queue = asyncio.Queue(1000) - -# # while True: -# async def nostr_subscribe(): -# return -# # for the relays: -# # async with websockets.connect("ws://localhost:8765") as websocket: -# # for the public keys: -# # await websocket.send("subscribe to events") -# # await websocket.recv() - -# ##################################################################### -# ################### LNBITS WEBSOCKET ROUTES ######################### -# #### HERE IS WHERE LNBITS FRONTEND CAN RECEIVE AND SEND MESSAGES #### -# ##################################################################### - -# class ConnectionManager: -# def __init__(self): -# self.active_connections: List[WebSocket] = [] - -# async def connect(self, websocket: WebSocket, nostr_id: str): -# await websocket.accept() -# websocket.id = nostr_id -# self.active_connections.append(websocket) - -# def disconnect(self, websocket: WebSocket): -# self.active_connections.remove(websocket) - -# async def send_personal_message(self, message: str, nostr_id: str): -# for connection in self.active_connections: -# if connection.id == nostr_id: -# await connection.send_text(message) - -# async def broadcast(self, message: str): -# for connection in self.active_connections: -# await connection.send_text(message) - - -# manager = ConnectionManager() - - -# @nostrclient_ext.websocket("/nostrclient/ws/relayevents/{nostr_id}", name="nostr_id.websocket_by_id") -# async def websocket_endpoint(websocket: WebSocket, nostr_id: str): -# await manager.connect(websocket, nostr_id) -# try: -# while True: -# data = await websocket.receive_text() -# except WebSocketDisconnect: -# manager.disconnect(websocket) - - -# async def updater(nostr_id, message): -# copilot = await get_copilot(nostr_id) -# if not copilot: -# return -# await manager.send_personal_message(f"{message}", nostr_id) - - -# async def relay_check(relay: str): -# async with websockets.connect(relay) as websocket: -# if str(websocket.state) == "State.OPEN": -# print(str(websocket.state)) -# return True -# else: -# return False diff --git a/views_api.py b/views_api.py index 1cf7a19..c3da200 100644 --- a/views_api.py +++ b/views_api.py @@ -11,8 +11,8 @@ from starlette.exceptions import HTTPException from . import nostrclient_ext from .crud import add_relay, delete_relay, get_relays from .models import Relay, RelayList -from .services import NostrRouter -from .tasks import client, init_relays +from .services import NostrRouter, nostr +from .tasks import init_relays # we keep this in all_routers: list[NostrRouter] = [] @@ -21,7 +21,7 @@ all_routers: list[NostrRouter] = [] @nostrclient_ext.get("/api/v1/relays") async def api_get_relays() -> RelayList: relays = RelayList(__root__=[]) - for url, r in client.relay_manager.relays.items(): + for url, r in nostr.client.relay_manager.relays.items(): status_text = ( f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" ) @@ -49,13 +49,14 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, detail=f"Relay url not provided." ) - if relay.url in client.relay_manager.relays: + if relay.url in nostr.client.relay_manager.relays: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, detail=f"Relay: {relay.url} already exists.", ) relay.id = urlsafe_short_hash() await add_relay(relay) + # we can't add relays during runtime yet await init_relays() return await get_relays() @@ -68,7 +69,8 @@ async def api_delete_relay(relay: Relay) -> None: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, detail=f"Relay url not provided." ) - client.relay_manager.remove_relay(relay.url) + # we can remove relays during runtime + nostr.client.relay_manager.remove_relay(relay.url) await delete_relay(relay) @@ -79,13 +81,13 @@ async def api_stop(): for router in all_routers: try: for s in router.subscriptions: - client.relay_manager.close_subscription(s) + nostr.client.relay_manager.close_subscription(s) await router.stop() all_routers.remove(router) except Exception as e: logger.error(e) try: - client.relay_manager.close_connections() + nostr.client.relay_manager.close_connections() except Exception as e: logger.error(e) @@ -106,7 +108,7 @@ async def ws_relay(websocket: WebSocket) -> None: if not router.connected: for s in router.subscriptions: try: - client.relay_manager.close_subscription(s) + nostr.client.relay_manager.close_subscription(s) except: pass await router.stop() From 2ff67b65e888f534a0a3d9e86e163bd56540aecd Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:31:32 +0100 Subject: [PATCH 04/96] fix it --- __init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/__init__.py b/__init__.py index 29f5658..90da4ec 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,3 @@ -import asyncio from fastapi import APIRouter from lnbits.db import Database from lnbits.helpers import template_renderer @@ -23,6 +22,8 @@ def nostr_renderer(): from .tasks import init_relays, subscribe_events +from .views import * # noqa +from .views_api import * # noqa def nostrclient_start(): From 619e0f05e25ae46487a06db82ab9d0cec7ba0f20 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 23 Mar 2023 10:35:45 +0200 Subject: [PATCH 05/96] chore: code format --- README.md | 2 -- __init__.py | 3 +- crud.py | 1 + models.py | 3 +- nostr/bech32.py | 33 ++++++++++++++------- nostr/client/cbc.py | 12 ++++---- nostr/client/client.py | 18 +++++------ nostr/delegation.py | 8 ++--- nostr/event.py | 7 +++-- nostr/key.py | 13 ++++---- nostr/message_pool.py | 3 +- nostr/message_type.py | 9 ++++-- nostr/pow.py | 11 +++++-- nostr/relay.py | 2 ++ nostr/subscription.py | 8 ++--- services.py | 4 +-- tasks.py | 6 ++-- templates/nostrclient/index.html | 51 +++++++++++++++++++++----------- views.py | 3 +- views_api.py | 5 ++-- 20 files changed, 121 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index e609b6c..5f9bfbc 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,4 @@ `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. - - ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) diff --git a/__init__.py b/__init__.py index 90da4ec..60d8e23 100644 --- a/__init__.py +++ b/__init__.py @@ -1,8 +1,9 @@ from fastapi import APIRouter +from starlette.staticfiles import StaticFiles + from lnbits.db import Database from lnbits.helpers import template_renderer from lnbits.tasks import catch_everything_and_restart -from starlette.staticfiles import StaticFiles db = Database("ext_nostrclient") diff --git a/crud.py b/crud.py index 497fcd7..780642d 100644 --- a/crud.py +++ b/crud.py @@ -1,6 +1,7 @@ from typing import List, Optional, Union import shortuuid + from lnbits.helpers import urlsafe_short_hash from . import db diff --git a/models.py b/models.py index 75a086d..4ed1e30 100644 --- a/models.py +++ b/models.py @@ -3,9 +3,10 @@ from typing import Dict, List, Optional from fastapi import Request from fastapi.param_functions import Query -from lnbits.helpers import urlsafe_short_hash from pydantic import BaseModel, Field +from lnbits.helpers import urlsafe_short_hash + class Relay(BaseModel): id: Optional[str] = None diff --git a/nostr/bech32.py b/nostr/bech32.py index b068de7..0ae6c80 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -23,21 +23,25 @@ from enum import Enum + class Encoding(Enum): """Enumeration type to list the various supported encodings.""" + BECH32 = 1 BECH32M = 2 + CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" -BECH32M_CONST = 0x2bc830a3 +BECH32M_CONST = 0x2BC830A3 + def bech32_polymod(values): """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] + generator = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] chk = 1 for value in values: top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value + chk = (chk & 0x1FFFFFF) << 5 ^ value for i in range(5): chk ^= generator[i] if ((top >> i) & 1) else 0 return chk @@ -57,6 +61,7 @@ def bech32_verify_checksum(hrp, data): return Encoding.BECH32M return None + def bech32_create_checksum(hrp, data, spec): """Compute the checksum values given HRP and data.""" values = bech32_hrp_expand(hrp) + data @@ -68,26 +73,29 @@ def bech32_create_checksum(hrp, data, spec): def bech32_encode(hrp, data, spec): """Compute a Bech32 string given HRP and data values.""" combined = data + bech32_create_checksum(hrp, data, spec) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) + return hrp + "1" + "".join([CHARSET[d] for d in combined]) + def bech32_decode(bech): """Validate a Bech32/Bech32m string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): + if (any(ord(x) < 33 or ord(x) > 126 for x in bech)) or ( + bech.lower() != bech and bech.upper() != bech + ): return (None, None, None) bech = bech.lower() - pos = bech.rfind('1') + pos = bech.rfind("1") if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: return (None, None, None) - if not all(x in CHARSET for x in bech[pos+1:]): + if not all(x in CHARSET for x in bech[pos + 1 :]): return (None, None, None) hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] + data = [CHARSET.find(x) for x in bech[pos + 1 :]] spec = bech32_verify_checksum(hrp, data) if spec is None: return (None, None, None) return (hrp, data[:-6], spec) + def convertbits(data, frombits, tobits, pad=True): """General power-of-2 base conversion.""" acc = 0 @@ -123,7 +131,12 @@ def decode(hrp, addr): return (None, None) if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: return (None, None) - if data[0] == 0 and spec != Encoding.BECH32 or data[0] != 0 and spec != Encoding.BECH32M: + if ( + data[0] == 0 + and spec != Encoding.BECH32 + or data[0] != 0 + and spec != Encoding.BECH32M + ): return (None, None) return (data[0], decoded) diff --git a/nostr/client/cbc.py b/nostr/client/cbc.py index a41dbc0..e69e8b5 100644 --- a/nostr/client/cbc.py +++ b/nostr/client/cbc.py @@ -1,4 +1,3 @@ - from Cryptodome import Random from Cryptodome.Cipher import AES @@ -11,10 +10,10 @@ key = bytes.fromhex("3aa925cb69eb613e2928f8a18279c78b1dca04541dfd064df2eda66b598 BLOCK_SIZE = 16 -class AESCipher(object): - """This class is compatible with crypto.createCipheriv('aes-256-cbc') - """ +class AESCipher(object): + """This class is compatible with crypto.createCipheriv('aes-256-cbc')""" + def __init__(self, key=None): self.key = key @@ -33,9 +32,10 @@ class AESCipher(object): def decrypt(self, iv, enc_text): cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) return self.unpad(cipher.decrypt(enc_text).decode("UTF-8")) - + + if __name__ == "__main__": aes = AESCipher(key=key) iv, enc_text = aes.encrypt(plain_text) dec_text = aes.decrypt(iv, enc_text) - print(dec_text) \ No newline at end of file + print(dec_text) diff --git a/nostr/client/client.py b/nostr/client/client.py index 6fb885f..bf91050 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,19 +1,15 @@ -from typing import * -import ssl -import time +import base64 import json import os -import base64 - -from ..event import Event -from ..relay_manager import RelayManager -from ..message_type import ClientMessageType -from ..key import PrivateKey, PublicKey +import ssl +import time +from typing import * +from ..event import EncryptedDirectMessage, Event, EventKind from ..filter import Filter, Filters -from ..event import Event, EventKind, EncryptedDirectMessage -from ..relay_manager import RelayManager +from ..key import PrivateKey, PublicKey from ..message_type import ClientMessageType +from ..relay_manager import RelayManager # from aes import AESCipher from . import cbc diff --git a/nostr/delegation.py b/nostr/delegation.py index 94801f5..8b1c311 100644 --- a/nostr/delegation.py +++ b/nostr/delegation.py @@ -7,23 +7,23 @@ class Delegation: delegator_pubkey: str delegatee_pubkey: str event_kind: int - duration_secs: int = 30*24*60 # default to 30 days + duration_secs: int = 30 * 24 * 60 # default to 30 days signature: str = None # set in PrivateKey.sign_delegation @property def expires(self) -> int: return int(time.time()) + self.duration_secs - + @property def conditions(self) -> str: return f"kind={self.event_kind}&created_at<{self.expires}" - + @property def delegation_token(self) -> str: return f"nostr:delegation:{self.delegatee_pubkey}:{self.conditions}" def get_tag(self) -> list[str]: - """ Called by Event """ + """Called by Event""" return [ "delegation", self.delegator_pubkey, diff --git a/nostr/event.py b/nostr/event.py index b903e0e..65b187d 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -1,10 +1,11 @@ -import time import json +import time from dataclasses import dataclass, field from enum import IntEnum -from typing import List -from secp256k1 import PublicKey from hashlib import sha256 +from typing import List + +from secp256k1 import PublicKey from .message_type import ClientMessageType diff --git a/nostr/key.py b/nostr/key.py index d34697f..8089e11 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,14 +1,15 @@ -import secrets import base64 -import secp256k1 -from cffi import FFI -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.primitives import padding +import secrets from hashlib import sha256 +import secp256k1 +from cffi import FFI +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +from . import bech32 from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind -from . import bech32 class PublicKey: diff --git a/nostr/message_pool.py b/nostr/message_pool.py index d364cf2..373d9fc 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -1,8 +1,9 @@ import json from queue import Queue from threading import Lock -from .message_type import RelayMessageType + from .event import Event +from .message_type import RelayMessageType class EventMessage: diff --git a/nostr/message_type.py b/nostr/message_type.py index 3f5206b..9f3be78 100644 --- a/nostr/message_type.py +++ b/nostr/message_type.py @@ -3,6 +3,7 @@ class ClientMessageType: REQUEST = "REQ" CLOSE = "CLOSE" + class RelayMessageType: EVENT = "EVENT" NOTICE = "NOTICE" @@ -10,6 +11,10 @@ class RelayMessageType: @staticmethod def is_valid(type: str) -> bool: - if type == RelayMessageType.EVENT or type == RelayMessageType.NOTICE or type == RelayMessageType.END_OF_STORED_EVENTS: + if ( + type == RelayMessageType.EVENT + or type == RelayMessageType.NOTICE + or type == RelayMessageType.END_OF_STORED_EVENTS + ): return True - return False \ No newline at end of file + return False diff --git a/nostr/pow.py b/nostr/pow.py index e006288..034ad9a 100644 --- a/nostr/pow.py +++ b/nostr/pow.py @@ -1,7 +1,9 @@ import time + from .event import Event from .key import PrivateKey + def zero_bits(b: int) -> int: n = 0 @@ -14,10 +16,11 @@ def zero_bits(b: int) -> int: return 7 - n + def count_leading_zero_bits(hex_str: str) -> int: total = 0 for i in range(0, len(hex_str) - 2, 2): - bits = zero_bits(int(hex_str[i:i+2], 16)) + bits = zero_bits(int(hex_str[i : i + 2], 16)) total += bits if bits != 8: @@ -25,7 +28,10 @@ def count_leading_zero_bits(hex_str: str) -> int: return total -def mine_event(content: str, difficulty: int, public_key: str, kind: int, tags: list=[]) -> Event: + +def mine_event( + content: str, difficulty: int, public_key: str, kind: int, tags: list = [] +) -> Event: all_tags = [["nonce", "1", str(difficulty)]] all_tags.extend(tags) @@ -43,6 +49,7 @@ def mine_event(content: str, difficulty: int, public_key: str, kind: int, tags: return Event(public_key, content, created_at, kind, all_tags, event_id) + def mine_key(difficulty: int) -> PrivateKey: sk = PrivateKey() num_leading_zero_bits = count_leading_zero_bits(sk.public_key.hex()) diff --git a/nostr/relay.py b/nostr/relay.py index ee78baa..0851c02 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,7 +2,9 @@ import json import time from queue import Queue from threading import Lock + from websocket import WebSocketApp + from .event import Event from .filter import Filters from .message_pool import MessagePool diff --git a/nostr/subscription.py b/nostr/subscription.py index 7afba20..10b5363 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,12 +1,10 @@ from .filter import Filters + class Subscription: - def __init__(self, id: str, filters: Filters=None) -> None: + def __init__(self, id: str, filters: Filters = None) -> None: self.id = id self.filters = filters def to_json_object(self): - return { - "id": self.id, - "filters": self.filters.to_json_array() - } + return {"id": self.id, "filters": self.filters.to_json_array()} diff --git a/services.py b/services.py index 09e9235..ab65dae 100644 --- a/services.py +++ b/services.py @@ -3,16 +3,16 @@ import json from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect + from lnbits.helpers import urlsafe_short_hash -from .nostr.client.client import NostrClient as NostrClientLib from .models import Event, Filter, Filters, Relay, RelayList +from .nostr.client.client import NostrClient as NostrClientLib from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage - received_subscription_events: dict[str, list[Event]] = {} received_subscription_notices: dict[str, list[NoticeMessage]] = {} received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} diff --git a/tasks.py b/tasks.py index 7894e40..1566f65 100644 --- a/tasks.py +++ b/tasks.py @@ -2,20 +2,18 @@ import asyncio import ssl import threading +from .crud import get_relays from .nostr.event import Event from .nostr.key import PublicKey from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage from .nostr.relay_manager import RelayManager from .services import ( nostr, - received_subscription_events, received_subscription_eosenotices, + received_subscription_events, ) -from .crud import get_relays - - async def init_relays(): # reinitialize the entire client nostr.__init__() diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 2a6cf60..b3d58c8 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -75,12 +75,17 @@ - -

Your endpoint: - -
+
+ Your endpoint: + +
@@ -88,7 +93,13 @@
- +
Add relay @@ -104,17 +115,21 @@
Nostrclient Extension

This extension is a always-on nostr client that other extensions can - use to send and receive events on nostr. - - Add multiple nostr relays to connect to. The extension then opens a websocket for you to use - at -

- - -

- Only Admin users can manage - this extension. + use to send and receive events on nostr. Add multiple nostr relays to + connect to. The extension then opens a websocket for you to use at

+ +

+ + +

+ Only Admin users can manage this extension. + @@ -217,7 +232,7 @@ message: `Invalid relay URL.`, caption: "Should start with 'wss://'' or 'ws://'" }) - return false; + return false } console.log('ADD RELAY ' + this.relayToAdd) let that = this @@ -229,7 +244,7 @@ {url: this.relayToAdd} ) .then(function (response) { - console.log("response:", response) + console.log('response:', response) if (response.data) { response.data.map(maplrelays) that.nostrrelayLinks = response.data @@ -239,7 +254,7 @@ .catch(function (error) { LNbits.utils.notifyApiError(error) }) - return false; + return false }, deleteRelay(url) { console.log('DELETE RELAY ' + url) diff --git a/views.py b/views.py index ce30b3b..a05f956 100644 --- a/views.py +++ b/views.py @@ -6,11 +6,12 @@ from fastapi import Request from fastapi.param_functions import Query from fastapi.params import Depends from fastapi.templating import Jinja2Templates +from starlette.responses import HTMLResponse + from lnbits.core.crud import update_payment_status from lnbits.core.models import User from lnbits.core.views.api import api_payment from lnbits.decorators import check_admin, check_user_exists -from starlette.responses import HTMLResponse from . import nostr_renderer, nostrclient_ext diff --git a/views_api.py b/views_api.py index c3da200..15cc3ab 100644 --- a/views_api.py +++ b/views_api.py @@ -3,11 +3,12 @@ from http import HTTPStatus from typing import Optional from fastapi import Depends, WebSocket -from lnbits.decorators import check_admin -from lnbits.helpers import urlsafe_short_hash from loguru import logger from starlette.exceptions import HTTPException +from lnbits.decorators import check_admin +from lnbits.helpers import urlsafe_short_hash + from . import nostrclient_ext from .crud import add_relay, delete_relay, get_relays from .models import Relay, RelayList From f369e2fdaccfb7f071829e26c8a060a555abe756 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 23 Mar 2023 10:57:29 +0200 Subject: [PATCH 06/96] Revert "chore: code format" This reverts commit 619e0f05e25ae46487a06db82ab9d0cec7ba0f20. --- README.md | 2 ++ __init__.py | 3 +- crud.py | 1 - models.py | 3 +- nostr/bech32.py | 33 +++++++-------------- nostr/client/cbc.py | 10 +++---- nostr/client/client.py | 20 +++++++------ nostr/delegation.py | 8 +++--- nostr/event.py | 5 ++-- nostr/key.py | 9 +++--- nostr/message_pool.py | 3 +- nostr/message_type.py | 9 ++---- nostr/pow.py | 11 ++----- nostr/relay.py | 2 -- nostr/subscription.py | 8 ++++-- services.py | 4 +-- tasks.py | 6 ++-- templates/nostrclient/index.html | 49 +++++++++++--------------------- views.py | 3 +- views_api.py | 5 ++-- 20 files changed, 77 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index 5f9bfbc..e609b6c 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,6 @@ `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. + + ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) diff --git a/__init__.py b/__init__.py index 60d8e23..90da4ec 100644 --- a/__init__.py +++ b/__init__.py @@ -1,9 +1,8 @@ from fastapi import APIRouter -from starlette.staticfiles import StaticFiles - from lnbits.db import Database from lnbits.helpers import template_renderer from lnbits.tasks import catch_everything_and_restart +from starlette.staticfiles import StaticFiles db = Database("ext_nostrclient") diff --git a/crud.py b/crud.py index 780642d..497fcd7 100644 --- a/crud.py +++ b/crud.py @@ -1,7 +1,6 @@ from typing import List, Optional, Union import shortuuid - from lnbits.helpers import urlsafe_short_hash from . import db diff --git a/models.py b/models.py index 4ed1e30..75a086d 100644 --- a/models.py +++ b/models.py @@ -3,9 +3,8 @@ from typing import Dict, List, Optional from fastapi import Request from fastapi.param_functions import Query -from pydantic import BaseModel, Field - from lnbits.helpers import urlsafe_short_hash +from pydantic import BaseModel, Field class Relay(BaseModel): diff --git a/nostr/bech32.py b/nostr/bech32.py index 0ae6c80..b068de7 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -23,25 +23,21 @@ from enum import Enum - class Encoding(Enum): """Enumeration type to list the various supported encodings.""" - BECH32 = 1 BECH32M = 2 - CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" -BECH32M_CONST = 0x2BC830A3 - +BECH32M_CONST = 0x2bc830a3 def bech32_polymod(values): """Internal function that computes the Bech32 checksum.""" - generator = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] + generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] chk = 1 for value in values: top = chk >> 25 - chk = (chk & 0x1FFFFFF) << 5 ^ value + chk = (chk & 0x1ffffff) << 5 ^ value for i in range(5): chk ^= generator[i] if ((top >> i) & 1) else 0 return chk @@ -61,7 +57,6 @@ def bech32_verify_checksum(hrp, data): return Encoding.BECH32M return None - def bech32_create_checksum(hrp, data, spec): """Compute the checksum values given HRP and data.""" values = bech32_hrp_expand(hrp) + data @@ -73,29 +68,26 @@ def bech32_create_checksum(hrp, data, spec): def bech32_encode(hrp, data, spec): """Compute a Bech32 string given HRP and data values.""" combined = data + bech32_create_checksum(hrp, data, spec) - return hrp + "1" + "".join([CHARSET[d] for d in combined]) - + return hrp + '1' + ''.join([CHARSET[d] for d in combined]) def bech32_decode(bech): """Validate a Bech32/Bech32m string, and determine HRP and data.""" - if (any(ord(x) < 33 or ord(x) > 126 for x in bech)) or ( - bech.lower() != bech and bech.upper() != bech - ): + if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or + (bech.lower() != bech and bech.upper() != bech)): return (None, None, None) bech = bech.lower() - pos = bech.rfind("1") + pos = bech.rfind('1') if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: return (None, None, None) - if not all(x in CHARSET for x in bech[pos + 1 :]): + if not all(x in CHARSET for x in bech[pos+1:]): return (None, None, None) hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos + 1 :]] + data = [CHARSET.find(x) for x in bech[pos+1:]] spec = bech32_verify_checksum(hrp, data) if spec is None: return (None, None, None) return (hrp, data[:-6], spec) - def convertbits(data, frombits, tobits, pad=True): """General power-of-2 base conversion.""" acc = 0 @@ -131,12 +123,7 @@ def decode(hrp, addr): return (None, None) if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: return (None, None) - if ( - data[0] == 0 - and spec != Encoding.BECH32 - or data[0] != 0 - and spec != Encoding.BECH32M - ): + if data[0] == 0 and spec != Encoding.BECH32 or data[0] != 0 and spec != Encoding.BECH32M: return (None, None) return (data[0], decoded) diff --git a/nostr/client/cbc.py b/nostr/client/cbc.py index e69e8b5..a41dbc0 100644 --- a/nostr/client/cbc.py +++ b/nostr/client/cbc.py @@ -1,3 +1,4 @@ + from Cryptodome import Random from Cryptodome.Cipher import AES @@ -10,10 +11,10 @@ key = bytes.fromhex("3aa925cb69eb613e2928f8a18279c78b1dca04541dfd064df2eda66b598 BLOCK_SIZE = 16 - class AESCipher(object): - """This class is compatible with crypto.createCipheriv('aes-256-cbc')""" + """This class is compatible with crypto.createCipheriv('aes-256-cbc') + """ def __init__(self, key=None): self.key = key @@ -32,10 +33,9 @@ class AESCipher(object): def decrypt(self, iv, enc_text): cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) return self.unpad(cipher.decrypt(enc_text).decode("UTF-8")) - - + if __name__ == "__main__": aes = AESCipher(key=key) iv, enc_text = aes.encrypt(plain_text) dec_text = aes.decrypt(iv, enc_text) - print(dec_text) + print(dec_text) \ No newline at end of file diff --git a/nostr/client/client.py b/nostr/client/client.py index bf91050..6fb885f 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,15 +1,19 @@ -import base64 -import json -import os +from typing import * import ssl import time -from typing import * +import json +import os +import base64 -from ..event import EncryptedDirectMessage, Event, EventKind -from ..filter import Filter, Filters -from ..key import PrivateKey, PublicKey -from ..message_type import ClientMessageType +from ..event import Event from ..relay_manager import RelayManager +from ..message_type import ClientMessageType +from ..key import PrivateKey, PublicKey + +from ..filter import Filter, Filters +from ..event import Event, EventKind, EncryptedDirectMessage +from ..relay_manager import RelayManager +from ..message_type import ClientMessageType # from aes import AESCipher from . import cbc diff --git a/nostr/delegation.py b/nostr/delegation.py index 8b1c311..94801f5 100644 --- a/nostr/delegation.py +++ b/nostr/delegation.py @@ -7,23 +7,23 @@ class Delegation: delegator_pubkey: str delegatee_pubkey: str event_kind: int - duration_secs: int = 30 * 24 * 60 # default to 30 days + duration_secs: int = 30*24*60 # default to 30 days signature: str = None # set in PrivateKey.sign_delegation @property def expires(self) -> int: return int(time.time()) + self.duration_secs - + @property def conditions(self) -> str: return f"kind={self.event_kind}&created_at<{self.expires}" - + @property def delegation_token(self) -> str: return f"nostr:delegation:{self.delegatee_pubkey}:{self.conditions}" def get_tag(self) -> list[str]: - """Called by Event""" + """ Called by Event """ return [ "delegation", self.delegator_pubkey, diff --git a/nostr/event.py b/nostr/event.py index 65b187d..b903e0e 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -1,11 +1,10 @@ -import json import time +import json from dataclasses import dataclass, field from enum import IntEnum -from hashlib import sha256 from typing import List - from secp256k1 import PublicKey +from hashlib import sha256 from .message_type import ClientMessageType diff --git a/nostr/key.py b/nostr/key.py index 8089e11..d34697f 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,15 +1,14 @@ -import base64 import secrets -from hashlib import sha256 - +import base64 import secp256k1 from cffi import FFI -from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives import padding +from hashlib import sha256 -from . import bech32 from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind +from . import bech32 class PublicKey: diff --git a/nostr/message_pool.py b/nostr/message_pool.py index 373d9fc..d364cf2 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -1,9 +1,8 @@ import json from queue import Queue from threading import Lock - -from .event import Event from .message_type import RelayMessageType +from .event import Event class EventMessage: diff --git a/nostr/message_type.py b/nostr/message_type.py index 9f3be78..3f5206b 100644 --- a/nostr/message_type.py +++ b/nostr/message_type.py @@ -3,7 +3,6 @@ class ClientMessageType: REQUEST = "REQ" CLOSE = "CLOSE" - class RelayMessageType: EVENT = "EVENT" NOTICE = "NOTICE" @@ -11,10 +10,6 @@ class RelayMessageType: @staticmethod def is_valid(type: str) -> bool: - if ( - type == RelayMessageType.EVENT - or type == RelayMessageType.NOTICE - or type == RelayMessageType.END_OF_STORED_EVENTS - ): + if type == RelayMessageType.EVENT or type == RelayMessageType.NOTICE or type == RelayMessageType.END_OF_STORED_EVENTS: return True - return False + return False \ No newline at end of file diff --git a/nostr/pow.py b/nostr/pow.py index 034ad9a..e006288 100644 --- a/nostr/pow.py +++ b/nostr/pow.py @@ -1,9 +1,7 @@ import time - from .event import Event from .key import PrivateKey - def zero_bits(b: int) -> int: n = 0 @@ -16,11 +14,10 @@ def zero_bits(b: int) -> int: return 7 - n - def count_leading_zero_bits(hex_str: str) -> int: total = 0 for i in range(0, len(hex_str) - 2, 2): - bits = zero_bits(int(hex_str[i : i + 2], 16)) + bits = zero_bits(int(hex_str[i:i+2], 16)) total += bits if bits != 8: @@ -28,10 +25,7 @@ def count_leading_zero_bits(hex_str: str) -> int: return total - -def mine_event( - content: str, difficulty: int, public_key: str, kind: int, tags: list = [] -) -> Event: +def mine_event(content: str, difficulty: int, public_key: str, kind: int, tags: list=[]) -> Event: all_tags = [["nonce", "1", str(difficulty)]] all_tags.extend(tags) @@ -49,7 +43,6 @@ def mine_event( return Event(public_key, content, created_at, kind, all_tags, event_id) - def mine_key(difficulty: int) -> PrivateKey: sk = PrivateKey() num_leading_zero_bits = count_leading_zero_bits(sk.public_key.hex()) diff --git a/nostr/relay.py b/nostr/relay.py index 0851c02..ee78baa 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,9 +2,7 @@ import json import time from queue import Queue from threading import Lock - from websocket import WebSocketApp - from .event import Event from .filter import Filters from .message_pool import MessagePool diff --git a/nostr/subscription.py b/nostr/subscription.py index 10b5363..7afba20 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,10 +1,12 @@ from .filter import Filters - class Subscription: - def __init__(self, id: str, filters: Filters = None) -> None: + def __init__(self, id: str, filters: Filters=None) -> None: self.id = id self.filters = filters def to_json_object(self): - return {"id": self.id, "filters": self.filters.to_json_array()} + return { + "id": self.id, + "filters": self.filters.to_json_array() + } diff --git a/services.py b/services.py index ab65dae..09e9235 100644 --- a/services.py +++ b/services.py @@ -3,16 +3,16 @@ import json from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect - from lnbits.helpers import urlsafe_short_hash +from .nostr.client.client import NostrClient as NostrClientLib from .models import Event, Filter, Filters, Relay, RelayList -from .nostr.client.client import NostrClient as NostrClientLib from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage + received_subscription_events: dict[str, list[Event]] = {} received_subscription_notices: dict[str, list[NoticeMessage]] = {} received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} diff --git a/tasks.py b/tasks.py index 1566f65..7894e40 100644 --- a/tasks.py +++ b/tasks.py @@ -2,18 +2,20 @@ import asyncio import ssl import threading -from .crud import get_relays from .nostr.event import Event from .nostr.key import PublicKey from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage from .nostr.relay_manager import RelayManager from .services import ( nostr, - received_subscription_eosenotices, received_subscription_events, + received_subscription_eosenotices, ) +from .crud import get_relays + + async def init_relays(): # reinitialize the entire client nostr.__init__() diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index b3d58c8..2a6cf60 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -75,17 +75,12 @@ + -
- Your endpoint: - -
+
Your endpoint: + +
@@ -93,13 +88,7 @@
- +
Add relay @@ -115,21 +104,17 @@
Nostrclient Extension

This extension is a always-on nostr client that other extensions can - use to send and receive events on nostr. Add multiple nostr relays to - connect to. The extension then opens a websocket for you to use at -

+ use to send and receive events on nostr. -

- - + Add multiple nostr relays to connect to. The extension then opens a websocket for you to use + at +

+ + +

+ Only Admin users can manage + this extension.

- Only Admin users can manage this extension. - @@ -232,7 +217,7 @@ message: `Invalid relay URL.`, caption: "Should start with 'wss://'' or 'ws://'" }) - return false + return false; } console.log('ADD RELAY ' + this.relayToAdd) let that = this @@ -244,7 +229,7 @@ {url: this.relayToAdd} ) .then(function (response) { - console.log('response:', response) + console.log("response:", response) if (response.data) { response.data.map(maplrelays) that.nostrrelayLinks = response.data @@ -254,7 +239,7 @@ .catch(function (error) { LNbits.utils.notifyApiError(error) }) - return false + return false; }, deleteRelay(url) { console.log('DELETE RELAY ' + url) diff --git a/views.py b/views.py index a05f956..ce30b3b 100644 --- a/views.py +++ b/views.py @@ -6,12 +6,11 @@ from fastapi import Request from fastapi.param_functions import Query from fastapi.params import Depends from fastapi.templating import Jinja2Templates -from starlette.responses import HTMLResponse - from lnbits.core.crud import update_payment_status from lnbits.core.models import User from lnbits.core.views.api import api_payment from lnbits.decorators import check_admin, check_user_exists +from starlette.responses import HTMLResponse from . import nostr_renderer, nostrclient_ext diff --git a/views_api.py b/views_api.py index 15cc3ab..c3da200 100644 --- a/views_api.py +++ b/views_api.py @@ -3,11 +3,10 @@ from http import HTTPStatus from typing import Optional from fastapi import Depends, WebSocket -from loguru import logger -from starlette.exceptions import HTTPException - from lnbits.decorators import check_admin from lnbits.helpers import urlsafe_short_hash +from loguru import logger +from starlette.exceptions import HTTPException from . import nostrclient_ext from .crud import add_relay, delete_relay, get_relays From 1e4ea75b00d95599262744d5c3a69217f2cc4834 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 23 Mar 2023 10:58:52 +0200 Subject: [PATCH 07/96] fix: exclude `/nostr` from code format --- README.md | 2 -- __init__.py | 3 +- crud.py | 1 + models.py | 3 +- services.py | 4 +-- tasks.py | 6 ++-- templates/nostrclient/index.html | 50 ++++++++++++++++++++------------ views.py | 3 +- views_api.py | 5 ++-- 9 files changed, 46 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e609b6c..5f9bfbc 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,4 @@ `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. - - ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) diff --git a/__init__.py b/__init__.py index 90da4ec..60d8e23 100644 --- a/__init__.py +++ b/__init__.py @@ -1,8 +1,9 @@ from fastapi import APIRouter +from starlette.staticfiles import StaticFiles + from lnbits.db import Database from lnbits.helpers import template_renderer from lnbits.tasks import catch_everything_and_restart -from starlette.staticfiles import StaticFiles db = Database("ext_nostrclient") diff --git a/crud.py b/crud.py index 497fcd7..780642d 100644 --- a/crud.py +++ b/crud.py @@ -1,6 +1,7 @@ from typing import List, Optional, Union import shortuuid + from lnbits.helpers import urlsafe_short_hash from . import db diff --git a/models.py b/models.py index 75a086d..4ed1e30 100644 --- a/models.py +++ b/models.py @@ -3,9 +3,10 @@ from typing import Dict, List, Optional from fastapi import Request from fastapi.param_functions import Query -from lnbits.helpers import urlsafe_short_hash from pydantic import BaseModel, Field +from lnbits.helpers import urlsafe_short_hash + class Relay(BaseModel): id: Optional[str] = None diff --git a/services.py b/services.py index 09e9235..ab65dae 100644 --- a/services.py +++ b/services.py @@ -3,16 +3,16 @@ import json from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect + from lnbits.helpers import urlsafe_short_hash -from .nostr.client.client import NostrClient as NostrClientLib from .models import Event, Filter, Filters, Relay, RelayList +from .nostr.client.client import NostrClient as NostrClientLib from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage - received_subscription_events: dict[str, list[Event]] = {} received_subscription_notices: dict[str, list[NoticeMessage]] = {} received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} diff --git a/tasks.py b/tasks.py index 7894e40..1566f65 100644 --- a/tasks.py +++ b/tasks.py @@ -2,20 +2,18 @@ import asyncio import ssl import threading +from .crud import get_relays from .nostr.event import Event from .nostr.key import PublicKey from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage from .nostr.relay_manager import RelayManager from .services import ( nostr, - received_subscription_events, received_subscription_eosenotices, + received_subscription_events, ) -from .crud import get_relays - - async def init_relays(): # reinitialize the entire client nostr.__init__() diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 2a6cf60..abe5a87 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -75,12 +75,17 @@ - -
Your endpoint: - -
+
+ Your endpoint: + +
@@ -88,7 +93,13 @@
- +
Add relay @@ -104,17 +115,20 @@
Nostrclient Extension

This extension is a always-on nostr client that other extensions can - use to send and receive events on nostr. - - Add multiple nostr relays to connect to. The extension then opens a websocket for you to use - at -

- - -

- Only Admin users can manage - this extension. + use to send and receive events on nostr. Add multiple nostr relays to + connect to. The extension then opens a websocket for you to use at

+ +

+ + +

+ Only Admin users can manage this extension. @@ -217,7 +231,7 @@ message: `Invalid relay URL.`, caption: "Should start with 'wss://'' or 'ws://'" }) - return false; + return false } console.log('ADD RELAY ' + this.relayToAdd) let that = this @@ -229,7 +243,7 @@ {url: this.relayToAdd} ) .then(function (response) { - console.log("response:", response) + console.log('response:', response) if (response.data) { response.data.map(maplrelays) that.nostrrelayLinks = response.data @@ -239,7 +253,7 @@ .catch(function (error) { LNbits.utils.notifyApiError(error) }) - return false; + return false }, deleteRelay(url) { console.log('DELETE RELAY ' + url) diff --git a/views.py b/views.py index ce30b3b..a05f956 100644 --- a/views.py +++ b/views.py @@ -6,11 +6,12 @@ from fastapi import Request from fastapi.param_functions import Query from fastapi.params import Depends from fastapi.templating import Jinja2Templates +from starlette.responses import HTMLResponse + from lnbits.core.crud import update_payment_status from lnbits.core.models import User from lnbits.core.views.api import api_payment from lnbits.decorators import check_admin, check_user_exists -from starlette.responses import HTMLResponse from . import nostr_renderer, nostrclient_ext diff --git a/views_api.py b/views_api.py index c3da200..15cc3ab 100644 --- a/views_api.py +++ b/views_api.py @@ -3,11 +3,12 @@ from http import HTTPStatus from typing import Optional from fastapi import Depends, WebSocket -from lnbits.decorators import check_admin -from lnbits.helpers import urlsafe_short_hash from loguru import logger from starlette.exceptions import HTTPException +from lnbits.decorators import check_admin +from lnbits.helpers import urlsafe_short_hash + from . import nostrclient_ext from .crud import add_relay, delete_relay, get_relays from .models import Relay, RelayList From e0938cb7601bf54eb94ab5328fa966031d4c7f0d Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 11 Apr 2023 15:47:21 +0200 Subject: [PATCH 08/96] resubscribe when a new relay is added --- README.md | 2 -- services.py | 4 ++-- tasks.py | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e609b6c..5f9bfbc 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,4 @@ `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. - - ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) diff --git a/services.py b/services.py index 09e9235..b1f5ad5 100644 --- a/services.py +++ b/services.py @@ -45,7 +45,6 @@ class NostrRouter: except WebSocketDisconnect: self.connected = False break - # print(json_str) # registers a subscription if the input was a REQ request subscription_id, json_str_rewritten = await self._add_nostr_subscription( @@ -81,7 +80,7 @@ class NostrRouter: } # this reconstructs the original response from the relay - # reconstruct oriiginal subscription id + # reconstruct original subscription id s_original = s[len(f"{self.subscription_id_rewrite}_") :] event_to_forward = ["EVENT", s_original, event_json] # print(json.dumps(event_to_forward)) @@ -104,6 +103,7 @@ class NostrRouter: async def stop(self): for t in self.tasks: t.cancel() + self.connected = False def _marshall_nostr_filters(self, data: Union[dict, list]): filters = data if isinstance(data, list) else [data] diff --git a/tasks.py b/tasks.py index 7894e40..7cee5bb 100644 --- a/tasks.py +++ b/tasks.py @@ -1,5 +1,6 @@ import asyncio import ssl +import json import threading from .nostr.event import Event @@ -17,6 +18,13 @@ from .crud import get_relays async def init_relays(): + # we save any subscriptions teporarily to re-add them after reinitializing the client + subscriptions = {} + for relay in nostr.client.relay_manager.relays.values(): + # relay.add_subscription(id, filters) + for subscription_id, filters in relay.subscriptions.items(): + subscriptions[subscription_id] = filters + # reinitialize the entire client nostr.__init__() # get relays from db @@ -24,6 +32,16 @@ async def init_relays(): # set relays and connect to them nostr.client.relays = list(set([r.url for r in relays.__root__ if r.url])) nostr.client.connect() + + await asyncio.sleep(2) + # re-add subscriptions + for subscription_id, subscription in subscriptions.items(): + nostr.client.relay_manager.add_subscription( + subscription_id, subscription.filters + ) + s = subscription.to_json_object() + json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) + nostr.client.relay_manager.publish_message(json_str) return From 38e5eeece02f71c5a78c812e0f69e244bc19c841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Mon, 17 Apr 2023 14:39:27 +0200 Subject: [PATCH 09/96] add the main tasks to scheduled_tasks and make dem deinitilize on api_stop --- __init__.py | 9 +++++++-- views_api.py | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index 60d8e23..5a251b0 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +from typing import List from fastapi import APIRouter from starlette.staticfiles import StaticFiles @@ -17,6 +18,8 @@ nostrclient_static_files = [ nostrclient_ext: APIRouter = APIRouter(prefix="/nostrclient", tags=["nostrclient"]) +scheduled_tasks: List[asyncio.Task] = [] + def nostr_renderer(): return template_renderer(["lnbits/extensions/nostrclient/templates"]) @@ -29,5 +32,7 @@ from .views_api import * # noqa def nostrclient_start(): loop = asyncio.get_event_loop() - loop.create_task(catch_everything_and_restart(init_relays)) - loop.create_task(catch_everything_and_restart(subscribe_events)) + task1 = loop.create_task(catch_everything_and_restart(init_relays)) + scheduled_tasks.append(task1) + task2 = loop.create_task(catch_everything_and_restart(subscribe_events)) + scheduled_tasks.append(task2) diff --git a/views_api.py b/views_api.py index 15cc3ab..c0be01e 100644 --- a/views_api.py +++ b/views_api.py @@ -9,7 +9,7 @@ from starlette.exceptions import HTTPException from lnbits.decorators import check_admin from lnbits.helpers import urlsafe_short_hash -from . import nostrclient_ext +from . import nostrclient_ext, scheduled_tasks from .crud import add_relay, delete_relay, get_relays from .models import Relay, RelayList from .services import NostrRouter, nostr @@ -92,6 +92,12 @@ async def api_stop(): except Exception as e: logger.error(e) + for scheduled_task in scheduled_tasks: + try: + scheduled_task.cancel() + except Exception as ex: + logger.warning(ex) + return {"success": True} From 6500e5c9bb2df62468ee7c099294eb1edff8fbb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Mon, 17 Apr 2023 14:42:14 +0200 Subject: [PATCH 10/96] forgot asyncio --- __init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/__init__.py b/__init__.py index 5a251b0..b09d0e9 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +import asyncio from typing import List from fastapi import APIRouter from starlette.staticfiles import StaticFiles From 625748e1199a593b23ee3828364dc733ee3ff2ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Mon, 17 Apr 2023 14:46:45 +0200 Subject: [PATCH 11/96] some import cleanup and wrong fastapi imports --- views.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/views.py b/views.py index a05f956..4214612 100644 --- a/views.py +++ b/views.py @@ -1,17 +1,9 @@ -import asyncio -from http import HTTPStatus - -# FastAPI good for incoming -from fastapi import Request -from fastapi.param_functions import Query -from fastapi.params import Depends +from fastapi import Request, Depends from fastapi.templating import Jinja2Templates from starlette.responses import HTMLResponse -from lnbits.core.crud import update_payment_status from lnbits.core.models import User -from lnbits.core.views.api import api_payment -from lnbits.decorators import check_admin, check_user_exists +from lnbits.decorators import check_admin from . import nostr_renderer, nostrclient_ext From 33df69c73ab1b97c428df491da2af57bc21c4067 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Apr 2023 15:47:03 +0200 Subject: [PATCH 12/96] fix endless loop --- nostr/client/client.py | 2 +- nostr/relay.py | 14 ++++++++++---- nostr/relay_manager.py | 20 +++++++++++++++----- services.py | 18 +++++++++++++++--- tasks.py | 5 ++++- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 6fb885f..6e70f71 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -141,7 +141,7 @@ class NostrClient: if callback_events_func: callback_events_func(event_msg) while self.relay_manager.message_pool.has_notices(): - event_msg = self.relay_manager.message_pool.has_notices() + event_msg = self.relay_manager.message_pool.get_notice() if callback_notices_func: callback_notices_func(event_msg) while self.relay_manager.message_pool.has_eose_notices(): diff --git a/nostr/relay.py b/nostr/relay.py index ee78baa..db9cacf 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -33,6 +33,7 @@ class Relay: self.subscriptions = subscriptions self.connected: bool = False self.reconnect: bool = True + self.shutdown: bool = False self.error_counter: int = 0 self.error_threshold: int = 0 self.num_received_events: int = 0 @@ -66,6 +67,7 @@ class Relay: def close(self): self.ws.close() + self.shutdown = True def check_reconnect(self): try: @@ -85,12 +87,16 @@ class Relay: def publish(self, message: str): self.queue.put(message) - def queue_worker(self): + def queue_worker(self, shutdown): while True: if self.connected: - message = self.queue.get() - self.num_sent_events += 1 - self.ws.send(message) + try: + message = self.queue.get(timeout=1) + self.num_sent_events += 1 + self.ws.send(message) + except: + if shutdown(): + break else: time.sleep(0.1) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 5b92d8d..a698a33 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -15,6 +15,8 @@ class RelayException(Exception): class RelayManager: def __init__(self) -> None: self.relays: dict[str, Relay] = {} + self.threads: dict[str, threading.Thread] = {} + self.queue_threads: dict[str, threading.Thread] = {} self.message_pool = MessagePool() def add_relay( @@ -25,7 +27,10 @@ class RelayManager: self.relays[url] = relay def remove_relay(self, url: str): + self.relays[url].close() self.relays.pop(url) + self.threads[url].join(timeout=1) + self.threads.pop(url) def add_subscription(self, id: str, filters: Filters): for relay in self.relays.values(): @@ -37,16 +42,21 @@ class RelayManager: def open_connections(self, ssl_options: dict = None, proxy: dict = None): for relay in self.relays.values(): - threading.Thread( + self.threads[relay.url] = threading.Thread( target=relay.connect, args=(ssl_options, proxy), name=f"{relay.url}-thread", daemon=True, - ).start() + ) + self.threads[relay.url].start() - threading.Thread( - target=relay.queue_worker, name=f"{relay.url}-queue", daemon=True - ).start() + self.queue_threads[relay.url] = threading.Thread( + target=relay.queue_worker, + args=(lambda: relay.shutdown,), + name=f"{relay.url}-queue", + daemon=True, + ) + self.queue_threads[relay.url].start() def close_connections(self): for relay in self.relays.values(): diff --git a/services.py b/services.py index e03ad1d..a801673 100644 --- a/services.py +++ b/services.py @@ -14,7 +14,7 @@ from .nostr.filter import Filters as NostrFilters from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage received_subscription_events: dict[str, list[Event]] = {} -received_subscription_notices: dict[str, list[NoticeMessage]] = {} +received_subscription_notices: list[NoticeMessage] = [] received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} @@ -62,7 +62,8 @@ class NostrRouter: stored in `my_subscriptions`. Then gets all responses for this subscription id from `received_subscription_events` which is filled in tasks.py. Takes one response after the other and relays it back to the client. Reconstructs the reponse manually because the nostr client lib we're using can't do it. Reconstructs the original subscription id - that we had previously rewritten in order to avoid collisions when multiple clients use the same id.""" + that we had previously rewritten in order to avoid collisions when multiple clients use the same id. + """ while True and self.connected: for s in self.subscriptions: if s in received_subscription_events: @@ -93,7 +94,17 @@ class NostrRouter: event_to_forward = ["EOSE", s_original] del received_subscription_eosenotices[s] # send data back to client + # print("Sending EOSE", event_to_forward) await self.websocket.send_text(json.dumps(event_to_forward)) + + # if s in received_subscription_notices: + while len(received_subscription_notices): + my_event = received_subscription_notices.pop(0) + event_to_forward = ["NOTICE", my_event.content] + # send data back to client + print("Received notice", event_to_forward) + # note: we don't send it to the user because we don't know who should receive it + # await self.websocket.send_text(json.dumps(event_to_forward)) await asyncio.sleep(0.1) async def start(self): @@ -128,7 +139,8 @@ class NostrRouter: """Parses a (string) request from a client. If it is a subscription (REQ), it will register the subscription in the nostr client library that we're using so we can receive the callbacks on it later. Will rewrite the subscription id since we expect - multiple clients to use the router and want to avoid subscription id collisions""" + multiple clients to use the router and want to avoid subscription id collisions + """ json_data = json.loads(json_str) assert len(json_data) if json_data[0] == "REQ": diff --git a/tasks.py b/tasks.py index 790337c..ab9a656 100644 --- a/tasks.py +++ b/tasks.py @@ -11,6 +11,7 @@ from .nostr.relay_manager import RelayManager from .services import ( nostr, received_subscription_eosenotices, + received_subscription_notices, received_subscription_events, ) @@ -68,7 +69,9 @@ async def subscribe_events(): ] return - def callback_notices(eventMessage: NoticeMessage): + def callback_notices(noticeMessage: NoticeMessage): + if noticeMessage not in received_subscription_notices: + received_subscription_notices.append(noticeMessage) return def callback_eose_notices(eventMessage: EndOfStoredEventsMessage): From ca30e730ab7db2b142215a5cd038cd1037a4e64a Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Apr 2023 16:06:15 +0200 Subject: [PATCH 13/96] rewrite CLOSE --- services.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services.py b/services.py index a801673..38bebca 100644 --- a/services.py +++ b/services.py @@ -50,6 +50,7 @@ class NostrRouter: subscription_id, json_str_rewritten = await self._add_nostr_subscription( json_str ) + if subscription_id and json_str_rewritten: self.subscriptions.append(subscription_id) json_str = json_str_rewritten @@ -84,6 +85,8 @@ class NostrRouter: # reconstruct original subscription id s_original = s[len(f"{self.subscription_id_rewrite}_") :] event_to_forward = ["EVENT", s_original, event_json] + + # print("Event to forward") # print(json.dumps(event_to_forward)) # send data back to client @@ -136,14 +139,14 @@ class NostrRouter: return NostrFilters(filter_list) async def _add_nostr_subscription(self, json_str): - """Parses a (string) request from a client. If it is a subscription (REQ), it will + """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will register the subscription in the nostr client library that we're using so we can receive the callbacks on it later. Will rewrite the subscription id since we expect multiple clients to use the router and want to avoid subscription id collisions """ json_data = json.loads(json_str) assert len(json_data) - if json_data[0] == "REQ": + if json_data[0] in ["REQ", "CLOSE"]: subscription_id = json_data[1] subscription_id_rewritten = ( f"{self.subscription_id_rewrite}_{subscription_id}" @@ -153,6 +156,8 @@ class NostrRouter: nostr.client.relay_manager.add_subscription( subscription_id_rewritten, filters ) - request_rewritten = json.dumps(["REQ", subscription_id_rewritten, fltr]) + request_rewritten = json.dumps( + [json_data[0], subscription_id_rewritten, fltr] + ) return subscription_id_rewritten, request_rewritten return None, None From 8aae5bdf3386d31e3bc8c0c133f004f7df0c21f1 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Apr 2023 16:08:16 +0200 Subject: [PATCH 14/96] loguru not print --- services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services.py b/services.py index 38bebca..2fc7b77 100644 --- a/services.py +++ b/services.py @@ -3,7 +3,7 @@ import json from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect - +from loguru import logger from lnbits.helpers import urlsafe_short_hash from .models import Event, Filter, Filters, Relay, RelayList @@ -105,7 +105,7 @@ class NostrRouter: my_event = received_subscription_notices.pop(0) event_to_forward = ["NOTICE", my_event.content] # send data back to client - print("Received notice", event_to_forward) + logger.debug("Nostrclient: Received notice", event_to_forward[1]) # note: we don't send it to the user because we don't know who should receive it # await self.websocket.send_text(json.dumps(event_to_forward)) await asyncio.sleep(0.1) From 4d4dd841b1454ea4683615229cab592aa00edb7a Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Apr 2023 18:18:03 +0200 Subject: [PATCH 15/96] replace subscription id parsing with a map --- services.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/services.py b/services.py index 2fc7b77..474bf90 100644 --- a/services.py +++ b/services.py @@ -32,7 +32,7 @@ class NostrRouter: self.connected: bool = True self.websocket = websocket self.tasks: List[asyncio.Task] = [] - self.subscription_id_rewrite: str = urlsafe_short_hash() + self.oridinal_subscription_ids = {} async def client_to_nostr(self): """Receives requests / data from the client and forwards it to relays. If the @@ -83,7 +83,7 @@ class NostrRouter: # this reconstructs the original response from the relay # reconstruct original subscription id - s_original = s[len(f"{self.subscription_id_rewrite}_") :] + s_original = self.oridinal_subscription_ids[s] event_to_forward = ["EVENT", s_original, event_json] # print("Event to forward") @@ -93,7 +93,7 @@ class NostrRouter: await self.websocket.send_text(json.dumps(event_to_forward)) if s in received_subscription_eosenotices: my_event = received_subscription_eosenotices[s] - s_original = s[len(f"{self.subscription_id_rewrite}_") :] + s_original = self.oridinal_subscription_ids[s] event_to_forward = ["EOSE", s_original] del received_subscription_eosenotices[s] # send data back to client @@ -148,9 +148,8 @@ class NostrRouter: assert len(json_data) if json_data[0] in ["REQ", "CLOSE"]: subscription_id = json_data[1] - subscription_id_rewritten = ( - f"{self.subscription_id_rewrite}_{subscription_id}" - ) + subscription_id_rewritten = urlsafe_short_hash() + self.oridinal_subscription_ids[subscription_id_rewritten] = subscription_id fltr = json_data[2] filters = self._marshall_nostr_filters(fltr) nostr.client.relay_manager.add_subscription( From 4c6fed4b10c55742a0a8b1f593d062a7e47c8f09 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 18 Apr 2023 17:47:17 +0200 Subject: [PATCH 16/96] create ws on connect --- nostr/relay.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index db9cacf..7595d9a 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -43,8 +43,10 @@ class Relay: self.proxy: dict = {} self.lock = Lock() self.queue = Queue() + + def connect(self, ssl_options: dict = None, proxy: dict = None): self.ws = WebSocketApp( - url, + self.url, on_open=self._on_open, on_message=self._on_message, on_error=self._on_error, @@ -52,8 +54,6 @@ class Relay: on_ping=self._on_ping, on_pong=self._on_pong, ) - - def connect(self, ssl_options: dict = None, proxy: dict = None): self.ssl_options = ssl_options self.proxy = proxy if not self.connected: From ea9fc18c151cd7fd500c8367be6f7952619cbb82 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 18 Apr 2023 17:50:41 +0200 Subject: [PATCH 17/96] reconnect on on_close --- nostr/relay.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 7595d9a..2649a4d 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -129,6 +129,10 @@ class Relay: def _on_close(self, class_obj, status_code, message): self.connected = False + if self.error_threshold and self.error_counter > self.error_threshold: + pass + else: + self.check_reconnect() pass def _on_message(self, class_obj, message: str): @@ -139,10 +143,6 @@ class Relay: def _on_error(self, class_obj, error): self.connected = False self.error_counter += 1 - if self.error_threshold and self.error_counter > self.error_threshold: - pass - else: - self.check_reconnect() def _on_ping(self, class_obj, message): return From 49f206eab305439a3105a43d7f0b607a14eea03d Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 18 Apr 2023 18:54:38 +0200 Subject: [PATCH 18/96] exponential delay for reconnect --- nostr/relay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 2649a4d..7fb4baa 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -35,7 +35,7 @@ class Relay: self.reconnect: bool = True self.shutdown: bool = False self.error_counter: int = 0 - self.error_threshold: int = 0 + self.error_threshold: int = 100 self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 @@ -76,7 +76,7 @@ class Relay: pass self.connected = False if self.reconnect: - time.sleep(1) + time.sleep(self.error_counter**2) self.connect(self.ssl_options, self.proxy) @property From 2814d277e5134d96b4732546c13a8eff0e90430a Mon Sep 17 00:00:00 2001 From: Tiago Vasconcelos Date: Mon, 24 Apr 2023 10:38:11 +0100 Subject: [PATCH 19/96] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f9bfbc..7fbb640 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# nostrclient +# Nostrclient - [LNbits](https://github.com/lnbits/lnbits) extension +For more about LNBits extension check [this tutorial](https://github.com/lnbits/lnbits/wiki/LNbits-Extensions) `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. From b5d98910db5ecfb53d8414e0197511e63d7477a8 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 10:12:49 +0300 Subject: [PATCH 20/96] chore: code format --- __init__.py | 1 + services.py | 1 + tasks.py | 4 ++-- views.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index b09d0e9..019df68 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,6 @@ import asyncio from typing import List + from fastapi import APIRouter from starlette.staticfiles import StaticFiles diff --git a/services.py b/services.py index 474bf90..82f6578 100644 --- a/services.py +++ b/services.py @@ -4,6 +4,7 @@ from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect from loguru import logger + from lnbits.helpers import urlsafe_short_hash from .models import Event, Filter, Filters, Relay, RelayList diff --git a/tasks.py b/tasks.py index ab9a656..beff9db 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,6 @@ import asyncio -import ssl import json +import ssl import threading from .crud import get_relays @@ -11,8 +11,8 @@ from .nostr.relay_manager import RelayManager from .services import ( nostr, received_subscription_eosenotices, - received_subscription_notices, received_subscription_events, + received_subscription_notices, ) diff --git a/views.py b/views.py index 4214612..57b73a1 100644 --- a/views.py +++ b/views.py @@ -1,4 +1,4 @@ -from fastapi import Request, Depends +from fastapi import Depends, Request from fastapi.templating import Jinja2Templates from starlette.responses import HTMLResponse From dc6c218618f19aa2c0e85489c46555b0918f6514 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 10:25:09 +0300 Subject: [PATCH 21/96] feat: prepare UI --- templates/nostrclient/index.html | 58 ++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index abe5a87..41fef48 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -2,6 +2,24 @@ %} {% block page %} {% raw %}
+ + +
+
+ +
+
+ Add relay +
+
+ +
@@ -42,7 +60,7 @@
{{ col.label }}
- + @@ -76,36 +94,24 @@ - -
- Your endpoint: - -
- + - - -
-
- +
+
+
+ Your endpoint: + +
-
- Add relay +
+ Test Endpoint
-
From 977ee84d9e661aedd626d9434da2fbd7ce0fff43 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 11:09:54 +0300 Subject: [PATCH 22/96] feat: basic test UI --- templates/nostrclient/index.html | 86 +++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 41fef48..4c63400 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -97,7 +97,8 @@ -
+ +
Your endpoint: @@ -112,6 +113,83 @@ Test Endpoint
+ + + +
+
+ Private Key (optional): +
+
+ +
+
+
+
+
+
+ + This should be a temp private (throw away). No not user your own private key! + + + + It is optional. One can be generated for you! + +
+ +
+
+
+ Test Message: +
+
+ +
+ +
+
+
+ Public Key (hex or npub): +
+
+ +
+
+
+
+
+
+ + This is the recipient of the message + +
+
+
+
+ Send Message +
+
+
@@ -126,7 +204,6 @@

- Date: Fri, 5 May 2023 11:10:23 +0300 Subject: [PATCH 23/96] feat: ass basic test api --- models.py | 10 ++++++++++ views_api.py | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/models.py b/models.py index 4ed1e30..899359f 100644 --- a/models.py +++ b/models.py @@ -50,6 +50,16 @@ class Filters(BaseModel): __root__: List[Filter] +class TestMessage(BaseModel): + sender_private_key: Optional[str] + reciever_public_key: str + message: str + +class TestMessageResponse(BaseModel): + private_key: str + public_key: str + event: Event + # class nostrKeys(BaseModel): # pubkey: str # privkey: str diff --git a/views_api.py b/views_api.py index c0be01e..3287351 100644 --- a/views_api.py +++ b/views_api.py @@ -11,7 +11,7 @@ from lnbits.helpers import urlsafe_short_hash from . import nostrclient_ext, scheduled_tasks from .crud import add_relay, delete_relay, get_relays -from .models import Relay, RelayList +from .models import Relay, RelayList, TestMessage, TestMessageResponse from .services import NostrRouter, nostr from .tasks import init_relays @@ -75,6 +75,26 @@ async def api_delete_relay(relay: Relay) -> None: await delete_relay(relay) +@nostrclient_ext.put( + "/api/v1/relay/test", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] +) +async def api_test_endpoint(test_message: TestMessage) -> TestMessageResponse: + try: + print("### api_test_endpoint", test_message) + except (ValueError, AssertionError) as ex: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail=str(ex), + ) + except Exception as ex: + logger.warning(ex) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Cannot generate test event", + ) + + + @nostrclient_ext.delete( "/api/v1", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] ) From 77f7c0b18f8a5b24c2bbff75ae305e9d77565013 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 11:12:12 +0300 Subject: [PATCH 24/96] chore: code format --- templates/nostrclient/index.html | 39 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 4c63400..593419a 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -15,7 +15,9 @@ >

- Add relay + Add relay +
@@ -60,7 +62,6 @@
{{ col.label }}
- @@ -94,7 +95,6 @@ - @@ -110,7 +110,9 @@
- Test Endpoint + Test Endpoint
@@ -131,22 +133,23 @@
-
-
+
- This should be a temp private (throw away). No not user your own private key! + This should be a temp private (throw away). No not user your + own private key! + - - + + It is optional. One can be generated for you!
-
- Test Message: + Test Message:
-
@@ -176,8 +178,7 @@
-
-
+
This is the recipient of the message @@ -186,7 +187,13 @@
- Send Message + Send Message
@@ -253,7 +260,7 @@ testData: { senderPrivateKey: null, recieverPublicKey: null, - message: null, + message: null }, relayTable: { columns: [ From 5d906c1fda17e13a52881f1dcb9c9c11eabf35fa Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:00:09 +0300 Subject: [PATCH 25/96] feat: send simple DM --- helpers.py | 19 +++++++ models.py | 2 +- templates/nostrclient/index.html | 88 ++++++++++++++++++++++++++++++-- views_api.py | 19 ++++++- 4 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 helpers.py diff --git a/helpers.py b/helpers.py new file mode 100644 index 0000000..bcf5c02 --- /dev/null +++ b/helpers.py @@ -0,0 +1,19 @@ +from bech32 import bech32_decode, convertbits + + +def normalize_public_key(pubkey: str) -> str: + if pubkey.startswith("npub1"): + _, decoded_data = bech32_decode(pubkey) + if not decoded_data: + raise ValueError("Public Key is not valid npub") + + decoded_data_bits = convertbits(decoded_data, 5, 8, False) + if not decoded_data_bits: + raise ValueError("Public Key is not valid npub") + return bytes(decoded_data_bits).hex() + + # check if valid hex + if len(pubkey) != 64: + raise ValueError("Public Key is not valid hex") + int(pubkey, 16) + return pubkey diff --git a/models.py b/models.py index 899359f..1456d83 100644 --- a/models.py +++ b/models.py @@ -58,7 +58,7 @@ class TestMessage(BaseModel): class TestMessageResponse(BaseModel): private_key: str public_key: str - event: Event + event_json: str # class nostrKeys(BaseModel): # pubkey: str diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 593419a..90859ae 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -159,7 +159,7 @@ filled rows="3" type="textarea" - label="Test Message" + label="Test Message *" >
@@ -181,7 +181,7 @@
- This is the recipient of the message + This is the recipient of the message. Field required.
@@ -189,6 +189,7 @@
+ + +
+
+ Sent Data: +
+
+ +
+
+
@@ -258,9 +278,12 @@ nostrrelayLinks: [], filter: '', testData: { + wsConnection: null, senderPrivateKey: null, recieverPublicKey: null, - message: null + message: null, + sentData: '', + receivedData: '' }, relayTable: { columns: [ @@ -368,6 +391,64 @@ LNbits.utils.notifyApiError(error) }) }, + sendTestMessage: async function(){ + try { + const {data} = await LNbits.api.request( + 'PUT', + '/nostrclient/api/v1/relay/test?usr=' + this.g.user.id, + this.g.user.wallets[0].adminkey, + { + sender_private_key: this.testData.senderPrivateKey, + reciever_public_key: this.testData.recieverPublicKey, + message: this.testData.message + } + ) + console.log('### data', data) + this.testData.senderPrivateKey = data.private_key + this.$q.localStorage.set('lnbits.nostrclient.senderPrivateKey', data.private_key || '') + const event = JSON.parse(data.event_json)[1] + console.log('### event', event) + this.sendDataToWebSocket(data.event_json) + } catch (error) { + LNbits.utils.notifyApiError(error) + } + }, + + sendDataToWebSocket: async function (data){ + try { + if (!this.testData.wsConnection) { + this.connectToWebsocket() + } + this.testData.wsConnection.send(data) + this.testData.sentData = data + '\n' + this.testData.sentData + } catch (error) { + this.$q.notify({ + timeout: 5000, + type: 'warning', + message: 'Failed to connect to websocket', + caption: `${error}` + }) + } + }, + connectToWebsocket: function () { + const scheme = location.protocol === 'http:' ? 'ws' : 'wss' + const port = location.port ? `:${location.port}` : '' + const wsUrl = `${scheme}://${document.domain}${port}/nostrclient/api/v1/relay` + this.testData.wsConnection = new WebSocket(wsUrl) + wsConnection.onmessage = async e => { + // const data = JSON.parse(e.data) + console.log('### onmessage', e.data) + } + wsConnection.onerror = async e => { + // const data = JSON.parse(e.data) + console.log('### onerror', e.data) + } + wsConnection.onclose = async e => { + // const data = JSON.parse(e.data) + console.log('### onclose', e.data) + } + + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) @@ -377,6 +458,7 @@ var self = this this.getRelays() setInterval(this.getRelays, 5000) + this.testData.senderPrivateKey = this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '' } }) diff --git a/views_api.py b/views_api.py index 3287351..e7c5f06 100644 --- a/views_api.py +++ b/views_api.py @@ -1,4 +1,5 @@ import asyncio +import json from http import HTTPStatus from typing import Optional @@ -11,7 +12,9 @@ from lnbits.helpers import urlsafe_short_hash from . import nostrclient_ext, scheduled_tasks from .crud import add_relay, delete_relay, get_relays +from .helpers import normalize_public_key from .models import Relay, RelayList, TestMessage, TestMessageResponse +from .nostr.key import EncryptedDirectMessage, PrivateKey from .services import NostrRouter, nostr from .tasks import init_relays @@ -78,9 +81,21 @@ async def api_delete_relay(relay: Relay) -> None: @nostrclient_ext.put( "/api/v1/relay/test", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] ) -async def api_test_endpoint(test_message: TestMessage) -> TestMessageResponse: +async def api_test_endpoint(data: TestMessage) -> TestMessageResponse: try: - print("### api_test_endpoint", test_message) + to_public_key = normalize_public_key(data.reciever_public_key) + + pk = bytes.fromhex(data.sender_private_key) if data.sender_private_key else None + private_key = PrivateKey(pk) + + dm = EncryptedDirectMessage( + recipient_pubkey=to_public_key, cleartext_content=data.message + ) + private_key.sign_event(dm) + + print("### api_test_endpoint", data) + + return TestMessageResponse(private_key=private_key.hex(), public_key=to_public_key, event_json=dm.to_message()) except (ValueError, AssertionError) as ex: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, From d74776ee9324898c2d1bfc27714aa1ef42dbc8a4 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:01:07 +0300 Subject: [PATCH 26/96] chore: code format --- templates/nostrclient/index.html | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 90859ae..acf50af 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -391,7 +391,7 @@ LNbits.utils.notifyApiError(error) }) }, - sendTestMessage: async function(){ + sendTestMessage: async function () { try { const {data} = await LNbits.api.request( 'PUT', @@ -405,7 +405,10 @@ ) console.log('### data', data) this.testData.senderPrivateKey = data.private_key - this.$q.localStorage.set('lnbits.nostrclient.senderPrivateKey', data.private_key || '') + this.$q.localStorage.set( + 'lnbits.nostrclient.senderPrivateKey', + data.private_key || '' + ) const event = JSON.parse(data.event_json)[1] console.log('### event', event) this.sendDataToWebSocket(data.event_json) @@ -414,14 +417,14 @@ } }, - sendDataToWebSocket: async function (data){ + sendDataToWebSocket: async function (data) { try { - if (!this.testData.wsConnection) { - this.connectToWebsocket() - } - this.testData.wsConnection.send(data) - this.testData.sentData = data + '\n' + this.testData.sentData - } catch (error) { + if (!this.testData.wsConnection) { + this.connectToWebsocket() + } + this.testData.wsConnection.send(data) + this.testData.sentData = data + '\n' + this.testData.sentData + } catch (error) { this.$q.notify({ timeout: 5000, type: 'warning', @@ -447,8 +450,7 @@ // const data = JSON.parse(e.data) console.log('### onclose', e.data) } - - }, + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) @@ -458,7 +460,9 @@ var self = this this.getRelays() setInterval(this.getRelays, 5000) - this.testData.senderPrivateKey = this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '' + this.testData.senderPrivateKey = + this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || + '' } }) From 58772043d4c874eb0a6d35130cba1d18caf386c8 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:45:45 +0300 Subject: [PATCH 27/96] feat: show sent data --- templates/nostrclient/index.html | 43 +++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index acf50af..27a2e08 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -216,6 +216,22 @@ >
+
+
+ Received Data: +
+
+ +
+
@@ -411,7 +427,7 @@ ) const event = JSON.parse(data.event_json)[1] console.log('### event', event) - this.sendDataToWebSocket(data.event_json) + await this.sendDataToWebSocket(data.event_json) } catch (error) { LNbits.utils.notifyApiError(error) } @@ -421,9 +437,11 @@ try { if (!this.testData.wsConnection) { this.connectToWebsocket() + await this.sleep(500) } this.testData.wsConnection.send(data) - this.testData.sentData = data + '\n' + this.testData.sentData + const separator = '='.repeat(80) + this.testData.sentData = data + `\n\n${separator}\n` + this.testData.sentData } catch (error) { this.$q.notify({ timeout: 5000, @@ -438,23 +456,20 @@ const port = location.port ? `:${location.port}` : '' const wsUrl = `${scheme}://${document.domain}${port}/nostrclient/api/v1/relay` this.testData.wsConnection = new WebSocket(wsUrl) - wsConnection.onmessage = async e => { - // const data = JSON.parse(e.data) - console.log('### onmessage', e.data) - } - wsConnection.onerror = async e => { - // const data = JSON.parse(e.data) - console.log('### onerror', e.data) - } - wsConnection.onclose = async e => { - // const data = JSON.parse(e.data) - console.log('### onclose', e.data) + const updateReciveData = async e => { + console.log('### updateReciveData', e.data) + this.testData.receivedData = e.data + '\n' +this.testData.receivedData } + + this.testData.wsConnection.onmessage = updateReciveData + this.testData.wsConnection.onerror = updateReciveData + this.testData.wsConnection.onclose = updateReciveData }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) - } + }, + sleep: (ms) => new Promise(r => setTimeout(r, ms)) }, created: function () { var self = this From 81e0aa26f8ef424c7fb527e22bbe3eb06b57b37d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:58:43 +0300 Subject: [PATCH 28/96] feat: subscribe to DMs --- templates/nostrclient/index.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 27a2e08..2d5a92d 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -210,7 +210,6 @@ v-model="testData.sentData" dense filled - readonly rows="5" type="textarea" > @@ -226,7 +225,6 @@ v-model="testData.receivedData" dense filled - readonly rows="5" type="textarea" > @@ -428,6 +426,8 @@ const event = JSON.parse(data.event_json)[1] console.log('### event', event) await this.sendDataToWebSocket(data.event_json) + const subscription = JSON.stringify(["REQ", "test-dms", { "kinds": [4], "#p": [event.pubkey]}]) + this.testData.wsConnection.send(subscription) } catch (error) { LNbits.utils.notifyApiError(error) } @@ -457,8 +457,8 @@ const wsUrl = `${scheme}://${document.domain}${port}/nostrclient/api/v1/relay` this.testData.wsConnection = new WebSocket(wsUrl) const updateReciveData = async e => { - console.log('### updateReciveData', e.data) - this.testData.receivedData = e.data + '\n' +this.testData.receivedData + const separator = '='.repeat(80) + this.testData.receivedData = e.data + `\n\n${separator}\n` +this.testData.receivedData } this.testData.wsConnection.onmessage = updateReciveData From d2e1dcc2ee382ea5b02337a2d95e3785202a27a6 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 14:00:03 +0300 Subject: [PATCH 29/96] chore: code cleanup & format --- templates/nostrclient/index.html | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 2d5a92d..78dd952 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -417,16 +417,18 @@ message: this.testData.message } ) - console.log('### data', data) this.testData.senderPrivateKey = data.private_key this.$q.localStorage.set( 'lnbits.nostrclient.senderPrivateKey', data.private_key || '' ) const event = JSON.parse(data.event_json)[1] - console.log('### event', event) await this.sendDataToWebSocket(data.event_json) - const subscription = JSON.stringify(["REQ", "test-dms", { "kinds": [4], "#p": [event.pubkey]}]) + const subscription = JSON.stringify([ + 'REQ', + 'test-dms', + {kinds: [4], '#p': [event.pubkey]} + ]) this.testData.wsConnection.send(subscription) } catch (error) { LNbits.utils.notifyApiError(error) @@ -441,7 +443,8 @@ } this.testData.wsConnection.send(data) const separator = '='.repeat(80) - this.testData.sentData = data + `\n\n${separator}\n` + this.testData.sentData + this.testData.sentData = + data + `\n\n${separator}\n` + this.testData.sentData } catch (error) { this.$q.notify({ timeout: 5000, @@ -458,9 +461,10 @@ this.testData.wsConnection = new WebSocket(wsUrl) const updateReciveData = async e => { const separator = '='.repeat(80) - this.testData.receivedData = e.data + `\n\n${separator}\n` +this.testData.receivedData + this.testData.receivedData = + e.data + `\n\n${separator}\n` + this.testData.receivedData } - + this.testData.wsConnection.onmessage = updateReciveData this.testData.wsConnection.onerror = updateReciveData this.testData.wsConnection.onclose = updateReciveData @@ -469,7 +473,7 @@ var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) }, - sleep: (ms) => new Promise(r => setTimeout(r, ms)) + sleep: ms => new Promise(r => setTimeout(r, ms)) }, created: function () { var self = this From b748dc3cb0ac604f81723bde18ab931dd404d569 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 14:27:07 +0300 Subject: [PATCH 30/96] chore: code clean-up --- templates/nostrclient/index.html | 80 +++++++++++++++++++++++++++----- views_api.py | 2 - 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 78dd952..bbd8f23 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -110,17 +110,17 @@
- Test Endpoint
- - + +
- Private Key (optional): + Sender Private Key:
+
+
+ Sender Public Key: +
+
+ +
+
Test Message: @@ -165,7 +179,7 @@
- Public Key (hex or npub): + Receiver Public Key:
@@ -198,8 +212,8 @@
- - + +
Sent Data: @@ -292,8 +306,10 @@ nostrrelayLinks: [], filter: '', testData: { + show: false, wsConnection: null, senderPrivateKey: null, + senderPublicKey: null, recieverPublicKey: null, message: null, sentData: '', @@ -405,6 +421,38 @@ LNbits.utils.notifyApiError(error) }) }, + toggleTestPanel: async function() { + if (this.testData.show) { + await this.hideTestPannel() + } else { + await this.showTestPanel() + } + }, + showTestPanel: async function() { + this.testData = { + show: true, + wsConnection: null, + senderPrivateKey: this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '', + recieverPublicKey: null, + message: null, + sentData: '', + receivedData: '' + } + await this.closeWebsocket() + this.connectToWebsocket() + }, + hideTestPannel: async function() { + await this.closeWebsocket() + this.testData = { + show: false, + wsConnection: null, + senderPrivateKey: null, + recieverPublicKey: null, + message: null, + sentData: '', + receivedData: '' + } + }, sendTestMessage: async function () { try { const {data} = await LNbits.api.request( @@ -423,6 +471,7 @@ data.private_key || '' ) const event = JSON.parse(data.event_json)[1] + this.testData.senderPublicKey = event.pubkey await this.sendDataToWebSocket(data.event_json) const subscription = JSON.stringify([ 'REQ', @@ -469,6 +518,16 @@ this.testData.wsConnection.onerror = updateReciveData this.testData.wsConnection.onclose = updateReciveData }, + closeWebsocket: async function () { + try { + if (this.testData.wsConnection) { + this.testData.wsConnection.close() + await this.sleep(100) + } + } catch (error) { + console.warn(error) + } + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) @@ -479,9 +538,6 @@ var self = this this.getRelays() setInterval(this.getRelays, 5000) - this.testData.senderPrivateKey = - this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || - '' } }) diff --git a/views_api.py b/views_api.py index e7c5f06..12f4f79 100644 --- a/views_api.py +++ b/views_api.py @@ -93,8 +93,6 @@ async def api_test_endpoint(data: TestMessage) -> TestMessageResponse: ) private_key.sign_event(dm) - print("### api_test_endpoint", data) - return TestMessageResponse(private_key=private_key.hex(), public_key=to_public_key, event_json=dm.to_message()) except (ValueError, AssertionError) as ex: raise HTTPException( From ebe16b8993c852b1fd3b11737eb924debdcadc14 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 8 May 2023 11:57:56 +0300 Subject: [PATCH 31/96] doc: add Troubleshoot --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 7fbb640..1de61ba 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,12 @@ `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) + + +### Troubleshoot +The `Test Endpoint` functionality heps the user to check that the `nostrclient` web-socket endpoint works as expected. + +The LNbits user can DM itself (or a temp account) from `nostrclient` and verify that the messages are sent and received correctly. + +https://user-images.githubusercontent.com/2951406/236780745-929c33c2-2502-49be-84a3-db02a7aabc0e.mp4 + From 55b84de873cb1b1e18c9edefe10889a3efac17be Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 8 May 2023 12:16:41 +0200 Subject: [PATCH 32/96] expansion button and copy button --- templates/nostrclient/index.html | 281 ++++++++++++++++--------------- 1 file changed, 147 insertions(+), 134 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index bbd8f23..3db09f3 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -99,8 +99,18 @@
-
+
+ Copy address Your endpoint:
-
- -
- - -
-
- Sender Private Key: + + + +
+
+ Sender Private Key: +
+
+ +
-
- +
+
+
+ + + No not use your real private key! Leave empty for a randomly + generated key. + +
-
-
-
-
- - This should be a temp private (throw away). No not user your - own private key! - - +
+
+ Sender Public Key: +
+
+ +
+
+
+
+ Test Message: +
+
+ +
+
+
+
+ Receiver Public Key: +
+
+ +
+
+
+
+
+ + This is the recipient of the message. Field required. + +
+
+
+
+ Send Message +
+
+ - - It is optional. One can be generated for you! - + + +
+
+ Sent Data: +
+
+ +
-
-
-
- Sender Public Key: +
+
+ Received Data: +
+
+ +
-
- -
-
-
-
- Test Message: -
-
- -
-
-
-
- Receiver Public Key: -
-
- -
-
-
-
-
- - This is the recipient of the message. Field required. - -
-
-
-
- Send Message -
-
- - - -
-
- Sent Data: -
-
- -
-
-
-
- Received Data: -
-
- -
-
- + +
@@ -421,18 +431,21 @@ LNbits.utils.notifyApiError(error) }) }, - toggleTestPanel: async function() { + toggleTestPanel: async function () { if (this.testData.show) { await this.hideTestPannel() } else { await this.showTestPanel() } }, - showTestPanel: async function() { + showTestPanel: async function () { this.testData = { show: true, wsConnection: null, - senderPrivateKey: this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '', + senderPrivateKey: + this.$q.localStorage.getItem( + 'lnbits.nostrclient.senderPrivateKey' + ) || '', recieverPublicKey: null, message: null, sentData: '', @@ -441,7 +454,7 @@ await this.closeWebsocket() this.connectToWebsocket() }, - hideTestPannel: async function() { + hideTestPannel: async function () { await this.closeWebsocket() this.testData = { show: false, From f27b4fa56962245b8abc37ef051c0bfe892c40b8 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 8 May 2023 12:29:43 +0200 Subject: [PATCH 33/96] readd toggletestbutton click --- templates/nostrclient/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 3db09f3..10fd4a5 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -125,6 +125,7 @@ group="advanced" icon="settings" label="Test this endpoint" + @click="toggleTestPanel" > From 629aa3a6c39de618ecb17fa5ecfe9b98103d8a85 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 11:54:04 +0300 Subject: [PATCH 34/96] fix: event uniqueness (see comment in code) --- nostr/message_pool.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nostr/message_pool.py b/nostr/message_pool.py index d364cf2..578d673 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -1,8 +1,9 @@ import json from queue import Queue from threading import Lock -from .message_type import RelayMessageType + from .event import Event +from .message_type import RelayMessageType class EventMessage: @@ -69,9 +70,18 @@ class MessagePool: ) with self.lock: if not event.id in self._unique_events: - self.events.put(EventMessage(event, subscription_id, url)) - self._unique_events.add(event.id) + self._accept_event(EventMessage(event, subscription_id, url)) elif message_type == RelayMessageType.NOTICE: self.notices.put(NoticeMessage(message_json[1], url)) elif message_type == RelayMessageType.END_OF_STORED_EVENTS: self.eose_notices.put(EndOfStoredEventsMessage(message_json[1], url)) + + def _accept_event(self, event_message: EventMessage): + """ + Event uniqueness is considered per `subscription_id`. + The `subscription_id` is rewritten to be unique and it is the same accross relays. + The same event can come from different subscriptions (from the same client or from different ones). + Clients that have joined later should receive older events. + """ + self.events.put(event_message) + self._unique_events.add(f"{event_message.subscription_id}_{event_message.event.id}") \ No newline at end of file From 6852a9aa5ef25688eff7443b9c9e6f233a79ac27 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 11:55:41 +0300 Subject: [PATCH 35/96] fix: do not share the `subscriptions` object between relays Change in one relay reflects in others. The RelayManager takes care of updating each relay individually. --- nostr/relay_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index a698a33..fb5839f 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -23,7 +23,7 @@ class RelayManager: self, url: str, read: bool = True, write: bool = True, subscriptions={} ): policy = RelayPolicy(read, write) - relay = Relay(url, policy, self.message_pool, subscriptions) + relay = Relay(url, policy, self.message_pool, subscriptions.copy()) self.relays[url] = relay def remove_relay(self, url: str): From 727f8fc3ce90451c1173fbca2165fa1bb1e2c695 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 11:55:56 +0300 Subject: [PATCH 36/96] fix: correctly handle `"REQ"` --- services.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/services.py b/services.py index 82f6578..b270539 100644 --- a/services.py +++ b/services.py @@ -48,16 +48,16 @@ class NostrRouter: break # registers a subscription if the input was a REQ request - subscription_id, json_str_rewritten = await self._add_nostr_subscription( + subscription_id, json_str_rewritten = await self._handle_nostr_subscription( json_str ) if subscription_id and json_str_rewritten: self.subscriptions.append(subscription_id) - json_str = json_str_rewritten # publish data - nostr.client.relay_manager.publish_message(json_str) + publish_data = json_str_rewritten or json_str + nostr.client.relay_manager.publish_message(publish_data) async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -139,7 +139,7 @@ class NostrRouter: ) return NostrFilters(filter_list) - async def _add_nostr_subscription(self, json_str): + async def _handle_nostr_subscription(self, json_str): """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will register the subscription in the nostr client library that we're using so we can receive the callbacks on it later. Will rewrite the subscription id since we expect @@ -147,7 +147,7 @@ class NostrRouter: """ json_data = json.loads(json_str) assert len(json_data) - if json_data[0] in ["REQ", "CLOSE"]: + if json_data[0] == "REQ": subscription_id = json_data[1] subscription_id_rewritten = urlsafe_short_hash() self.oridinal_subscription_ids[subscription_id_rewritten] = subscription_id @@ -160,4 +160,12 @@ class NostrRouter: [json_data[0], subscription_id_rewritten, fltr] ) return subscription_id_rewritten, request_rewritten + elif json_data[0] == "CLOSE": + subscription_id = json_data[1] + subscription_id_rewritten = next((k for k, v in self.oridinal_subscription_ids.items() if v == subscription_id), None) + if subscription_id_rewritten: + nostr.client.relay_manager.close_subscription(subscription_id_rewritten) + request_rewritten = json.dumps([json_data[0], subscription_id_rewritten]) + return None, request_rewritten + return None, None From 7e2c9033346cb969c674801cda45dc7cfd968eaa Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 12:13:57 +0300 Subject: [PATCH 37/96] fix: uniqueness check --- nostr/message_pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/message_pool.py b/nostr/message_pool.py index 578d673..02f7fd4 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -69,7 +69,7 @@ class MessagePool: e["sig"], ) with self.lock: - if not event.id in self._unique_events: + if not f"{subscription_id}_{event.id}" in self._unique_events: self._accept_event(EventMessage(event, subscription_id, url)) elif message_type == RelayMessageType.NOTICE: self.notices.put(NoticeMessage(message_json[1], url)) From 322679e7c5fe07b1004f6fb910e304dc81947651 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 21 Jun 2023 14:40:28 +0300 Subject: [PATCH 38/96] fix: do not use lambda in a loop --- nostr/pow.py | 2 ++ nostr/relay.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nostr/pow.py b/nostr/pow.py index e006288..fece484 100644 --- a/nostr/pow.py +++ b/nostr/pow.py @@ -1,7 +1,9 @@ import time + from .event import Event from .key import PrivateKey + def zero_bits(b: int) -> int: n = 0 diff --git a/nostr/relay.py b/nostr/relay.py index 7fb4baa..246b985 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,7 +2,9 @@ import json import time from queue import Queue from threading import Lock + from websocket import WebSocketApp + from .event import Event from .filter import Filters from .message_pool import MessagePool @@ -45,6 +47,7 @@ class Relay: self.queue = Queue() def connect(self, ssl_options: dict = None, proxy: dict = None): + print("### relay.connect", self.url) self.ws = WebSocketApp( self.url, on_open=self._on_open, @@ -81,24 +84,29 @@ class Relay: @property def ping(self): + print("### ping: ", self.url) ping_ms = int((self.ws.last_pong_tm - self.ws.last_ping_tm) * 1000) return ping_ms if self.connected and ping_ms > 0 else 0 def publish(self, message: str): self.queue.put(message) - def queue_worker(self, shutdown): + def queue_worker(self): + print("#### IN !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) while True: if self.connected: try: message = self.queue.get(timeout=1) + print("#### queue_worker", message) self.num_sent_events += 1 self.ws.send(message) - except: - if shutdown(): + except Exception as e: + if self.shutdown: + print("#### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! e [", e, self.url, self.shutdown," ]###") break else: time.sleep(0.1) + print("#### OUT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) def add_subscription(self, id, filters: Filters): with self.lock: From d08e91b2c7b7954386141ee2b242b3f56da55fe0 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 21 Jun 2023 17:12:26 +0300 Subject: [PATCH 39/96] fix: do not double re-connect --- nostr/relay_manager.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index fb5839f..f6eba36 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -22,6 +22,8 @@ class RelayManager: def add_relay( self, url: str, read: bool = True, write: bool = True, subscriptions={} ): + if url in self.relays: + return policy = RelayPolicy(read, write) relay = Relay(url, policy, self.message_pool, subscriptions.copy()) self.relays[url] = relay @@ -42,21 +44,23 @@ class RelayManager: def open_connections(self, ssl_options: dict = None, proxy: dict = None): for relay in self.relays.values(): - self.threads[relay.url] = threading.Thread( - target=relay.connect, - args=(ssl_options, proxy), - name=f"{relay.url}-thread", - daemon=True, - ) - self.threads[relay.url].start() + if relay.url not in self.threads: + self.threads[relay.url] = threading.Thread( + target=relay.connect, + args=(ssl_options, proxy), + name=f"{relay.url}-thread", + daemon=True, + ) - self.queue_threads[relay.url] = threading.Thread( - target=relay.queue_worker, - args=(lambda: relay.shutdown,), - name=f"{relay.url}-queue", - daemon=True, - ) - self.queue_threads[relay.url].start() + self.threads[relay.url].start() + + if relay.url not in self.queue_threads: + self.queue_threads[relay.url] = threading.Thread( + target=relay.queue_worker, + name=f"{relay.url}-queue", + daemon=True, + ) + self.queue_threads[relay.url].start() def close_connections(self): for relay in self.relays.values(): From 09d2fc0493d56e407fa4b2c36d7fa0ba7fa6f672 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 10:02:15 +0300 Subject: [PATCH 40/96] refactor: `add_relay` logic --- nostr/client/client.py | 27 +++++++++------------ nostr/relay.py | 6 +++++ nostr/relay_manager.py | 55 ++++++++++++++++++++++++------------------ tasks.py | 20 +-------------- views_api.py | 14 ++++++++--- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 6e70f71..4d10647 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,19 +1,15 @@ -from typing import * -import ssl -import time +import base64 import json import os -import base64 - -from ..event import Event -from ..relay_manager import RelayManager -from ..message_type import ClientMessageType -from ..key import PrivateKey, PublicKey +import time +from typing import * +from ..event import EncryptedDirectMessage, Event, EventKind from ..filter import Filter, Filters -from ..event import Event, EventKind, EncryptedDirectMessage -from ..relay_manager import RelayManager +from ..key import PrivateKey, PublicKey from ..message_type import ClientMessageType +from ..relay_manager import RelayManager +from ..subscription import Subscription # from aes import AESCipher from . import cbc @@ -38,12 +34,11 @@ class NostrClient: if connect: self.connect() - def connect(self): + async def connect(self, subscriptions: dict[str, Subscription] = {}): for relay in self.relays: - self.relay_manager.add_relay(relay) - self.relay_manager.open_connections( - {"cert_reqs": ssl.CERT_NONE} - ) # NOTE: This disables ssl certificate verification + self.relay_manager.add_relay(relay, subscriptions) + + def close(self): self.relay_manager.close_connections() diff --git a/nostr/relay.py b/nostr/relay.py index 246b985..b6207b5 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -91,6 +91,12 @@ class Relay: def publish(self, message: str): self.queue.put(message) + def publish_subscriptions(self): + for _, subscription in self.subscriptions.items(): + s = subscription.to_json_object() + json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) + self.publish(json_str) + def queue_worker(self): print("#### IN !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) while True: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index f6eba36..0ec324a 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,11 +1,12 @@ -import json + +import ssl import threading from .event import Event from .filter import Filters from .message_pool import MessagePool -from .message_type import ClientMessageType from .relay import Relay, RelayPolicy +from .subscription import Subscription class RelayException(Exception): @@ -20,19 +21,30 @@ class RelayManager: self.message_pool = MessagePool() def add_relay( - self, url: str, read: bool = True, write: bool = True, subscriptions={} - ): + self, url: str, read: bool = True, write: bool = True, subscriptions: dict[str, Subscription] = {} + ) -> Relay: if url in self.relays: return + policy = RelayPolicy(read, write) relay = Relay(url, policy, self.message_pool, subscriptions.copy()) self.relays[url] = relay + self.open_connection( + relay, + {"cert_reqs": ssl.CERT_NONE} + ) # NOTE: This disables ssl certificate verification + + relay.publish_subscriptions() + return relay + def remove_relay(self, url: str): - self.relays[url].close() - self.relays.pop(url) self.threads[url].join(timeout=1) self.threads.pop(url) + self.queue_threads[url].join(timeout=1) + self.queue_threads.pop(url) + self.relays[url].close() + self.relays.pop(url) def add_subscription(self, id: str, filters: Filters): for relay in self.relays.values(): @@ -42,25 +54,22 @@ class RelayManager: for relay in self.relays.values(): relay.close_subscription(id) - def open_connections(self, ssl_options: dict = None, proxy: dict = None): - for relay in self.relays.values(): - if relay.url not in self.threads: - self.threads[relay.url] = threading.Thread( - target=relay.connect, - args=(ssl_options, proxy), - name=f"{relay.url}-thread", - daemon=True, - ) - self.threads[relay.url].start() + def open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): + self.threads[relay.url] = threading.Thread( + target=relay.connect, + args=(ssl_options, proxy), + name=f"{relay.url}-thread", + daemon=True, + ) + self.threads[relay.url].start() - if relay.url not in self.queue_threads: - self.queue_threads[relay.url] = threading.Thread( - target=relay.queue_worker, - name=f"{relay.url}-queue", - daemon=True, - ) - self.queue_threads[relay.url].start() + self.queue_threads[relay.url] = threading.Thread( + target=relay.queue_worker, + name=f"{relay.url}-queue", + daemon=True, + ) + self.queue_threads[relay.url].start() def close_connections(self): for relay in self.relays.values(): diff --git a/tasks.py b/tasks.py index beff9db..eb5391a 100644 --- a/tasks.py +++ b/tasks.py @@ -17,31 +17,13 @@ from .services import ( async def init_relays(): - # we save any subscriptions teporarily to re-add them after reinitializing the client - subscriptions = {} - for relay in nostr.client.relay_manager.relays.values(): - # relay.add_subscription(id, filters) - for subscription_id, filters in relay.subscriptions.items(): - subscriptions[subscription_id] = filters - # reinitialize the entire client nostr.__init__() # get relays from db relays = await get_relays() # set relays and connect to them nostr.client.relays = list(set([r.url for r in relays.__root__ if r.url])) - nostr.client.connect() - - await asyncio.sleep(2) - # re-add subscriptions - for subscription_id, subscription in subscriptions.items(): - nostr.client.relay_manager.add_subscription( - subscription_id, subscription.filters - ) - s = subscription.to_json_object() - json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) - nostr.client.relay_manager.publish_message(json_str) - return + await nostr.client.connect() async def subscribe_events(): diff --git a/views_api.py b/views_api.py index 12f4f79..88193ad 100644 --- a/views_api.py +++ b/views_api.py @@ -1,7 +1,7 @@ import asyncio import json from http import HTTPStatus -from typing import Optional +from typing import List, Optional from fastapi import Depends, WebSocket from loguru import logger @@ -15,6 +15,7 @@ from .crud import add_relay, delete_relay, get_relays from .helpers import normalize_public_key from .models import Relay, RelayList, TestMessage, TestMessageResponse from .nostr.key import EncryptedDirectMessage, PrivateKey +from .nostr.relay import Relay as NostrRelay from .services import NostrRouter, nostr from .tasks import init_relays @@ -60,8 +61,15 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: ) relay.id = urlsafe_short_hash() await add_relay(relay) - # we can't add relays during runtime yet - await init_relays() + + all_relays: List[NostrRelay] = nostr.client.relay_manager.relays.values() + if len(all_relays): + subscriptions = all_relays[0].subscriptions + nostr.client.relays.append(relay.url) + nostr.client.relay_manager.add_relay(subscriptions) + + nostr.client.relay_manager.connect_relay(relay.url) + return await get_relays() From 1a58f0fea8a6340bbce2d910a05410efd2b326cb Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 10:02:29 +0300 Subject: [PATCH 41/96] chore: code format --- nostr/bech32.py | 1 + nostr/event.py | 7 ++++--- nostr/key.py | 13 +++++++------ nostr/subscription.py | 1 + 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/nostr/bech32.py b/nostr/bech32.py index b068de7..61a92c4 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -23,6 +23,7 @@ from enum import Enum + class Encoding(Enum): """Enumeration type to list the various supported encodings.""" BECH32 = 1 diff --git a/nostr/event.py b/nostr/event.py index b903e0e..65b187d 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -1,10 +1,11 @@ -import time import json +import time from dataclasses import dataclass, field from enum import IntEnum -from typing import List -from secp256k1 import PublicKey from hashlib import sha256 +from typing import List + +from secp256k1 import PublicKey from .message_type import ClientMessageType diff --git a/nostr/key.py b/nostr/key.py index d34697f..8089e11 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,14 +1,15 @@ -import secrets import base64 -import secp256k1 -from cffi import FFI -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.primitives import padding +import secrets from hashlib import sha256 +import secp256k1 +from cffi import FFI +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +from . import bech32 from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind -from . import bech32 class PublicKey: diff --git a/nostr/subscription.py b/nostr/subscription.py index 7afba20..76da0af 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,5 +1,6 @@ from .filter import Filters + class Subscription: def __init__(self, id: str, filters: Filters=None) -> None: self.id = id From 666009720a6b8189c8a85c49d2a828220f5801ec Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 10:22:13 +0300 Subject: [PATCH 42/96] chore: code clean-up --- nostr/client/client.py | 97 +----------------------------------------- nostr/relay.py | 5 --- tasks.py | 1 - 3 files changed, 1 insertion(+), 102 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 4d10647..faf5722 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,19 +1,9 @@ -import base64 -import json -import os import time from typing import * -from ..event import EncryptedDirectMessage, Event, EventKind -from ..filter import Filter, Filters -from ..key import PrivateKey, PublicKey -from ..message_type import ClientMessageType from ..relay_manager import RelayManager from ..subscription import Subscription -# from aes import AESCipher -from . import cbc - class NostrClient: relays = [ @@ -23,12 +13,8 @@ class NostrClient: "wss://nostr.oxtr.dev", ] # ["wss://nostr.oxtr.dev"] # ["wss://relay.nostr.info"] "wss://nostr-pub.wellorder.net" "ws://91.237.88.218:2700", "wss://nostrrr.bublina.eu.org", ""wss://nostr-relay.freeberty.net"", , "wss://nostr.oxtr.dev", "wss://relay.nostr.info", "wss://nostr-pub.wellorder.net" , "wss://relayer.fiatjaf.com", "wss://nodestr.fmt.wiz.biz/", "wss://no.str.cr" relay_manager = RelayManager() - private_key: PrivateKey - public_key: PublicKey - - def __init__(self, privatekey_hex: str = "", relays: List[str] = [], connect=True): - self.generate_keys(privatekey_hex) + def __init__(self, relays: List[str] = [], connect=True): if len(relays): self.relays = relays if connect: @@ -43,87 +29,6 @@ class NostrClient: def close(self): self.relay_manager.close_connections() - def generate_keys(self, privatekey_hex: str = None): - pk = bytes.fromhex(privatekey_hex) if privatekey_hex else None - self.private_key = PrivateKey(pk) - self.public_key = self.private_key.public_key - - def post(self, message: str): - event = Event(message, self.public_key.hex(), kind=EventKind.TEXT_NOTE) - self.private_key.sign_event(event) - event_json = event.to_message() - # print("Publishing message:") - # print(event_json) - self.relay_manager.publish_message(event_json) - - def get_post( - self, sender_publickey: PublicKey = None, callback_func=None, filter_kwargs={} - ): - filter = Filter( - authors=[sender_publickey.hex()] if sender_publickey else None, - kinds=[EventKind.TEXT_NOTE], - **filter_kwargs, - ) - filters = Filters([filter]) - subscription_id = os.urandom(4).hex() - self.relay_manager.add_subscription(subscription_id, filters) - - request = [ClientMessageType.REQUEST, subscription_id] - request.extend(filters.to_json_array()) - message = json.dumps(request) - self.relay_manager.publish_message(message) - - while True: - while self.relay_manager.message_pool.has_events(): - event_msg = self.relay_manager.message_pool.get_event() - if callback_func: - callback_func(event_msg.event) - time.sleep(0.1) - - def dm(self, message: str, to_pubkey: PublicKey): - dm = EncryptedDirectMessage( - recipient_pubkey=to_pubkey.hex(), cleartext_content=message - ) - self.private_key.sign_event(dm) - self.relay_manager.publish_event(dm) - - def get_dm(self, sender_publickey: PublicKey, callback_func=None): - filters = Filters( - [ - Filter( - kinds=[EventKind.ENCRYPTED_DIRECT_MESSAGE], - pubkey_refs=[sender_publickey.hex()], - ) - ] - ) - subscription_id = os.urandom(4).hex() - self.relay_manager.add_subscription(subscription_id, filters) - - request = [ClientMessageType.REQUEST, subscription_id] - request.extend(filters.to_json_array()) - message = json.dumps(request) - self.relay_manager.publish_message(message) - - while True: - while self.relay_manager.message_pool.has_events(): - event_msg = self.relay_manager.message_pool.get_event() - if "?iv=" in event_msg.event.content: - try: - shared_secret = self.private_key.compute_shared_secret( - event_msg.event.public_key - ) - aes = cbc.AESCipher(key=shared_secret) - enc_text_b64, iv_b64 = event_msg.event.content.split("?iv=") - iv = base64.decodebytes(iv_b64.encode("utf-8")) - enc_text = base64.decodebytes(enc_text_b64.encode("utf-8")) - dec_text = aes.decrypt(iv, enc_text) - if callback_func: - callback_func(event_msg.event, dec_text) - except: - pass - break - time.sleep(0.1) - def subscribe( self, callback_events_func=None, diff --git a/nostr/relay.py b/nostr/relay.py index b6207b5..94e532c 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -122,11 +122,6 @@ class Relay: with self.lock: self.subscriptions.pop(id) - def update_subscription(self, id: str, filters: Filters) -> None: - with self.lock: - subscription = self.subscriptions[id] - subscription.filters = filters - def to_json_object(self) -> dict: return { "url": self.url, diff --git a/tasks.py b/tasks.py index eb5391a..069c57d 100644 --- a/tasks.py +++ b/tasks.py @@ -31,7 +31,6 @@ async def subscribe_events(): await asyncio.sleep(2) def callback_events(eventMessage: EventMessage): - # print(f"From {event.public_key[:3]}..{event.public_key[-3:]}: {event.content}") if eventMessage.subscription_id in received_subscription_events: # do not add duplicate events (by event id) if eventMessage.event.id in set( From e2458d43dfdf1028d288810839f45fbc26e462a9 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:00 +0300 Subject: [PATCH 43/96] chore: code clean-up --- nostr/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index faf5722..0ff85a8 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -11,7 +11,7 @@ class NostrClient: "wss://nostr.zebedee.cloud", "wss://nodestr.fmt.wiz.biz", "wss://nostr.oxtr.dev", - ] # ["wss://nostr.oxtr.dev"] # ["wss://relay.nostr.info"] "wss://nostr-pub.wellorder.net" "ws://91.237.88.218:2700", "wss://nostrrr.bublina.eu.org", ""wss://nostr-relay.freeberty.net"", , "wss://nostr.oxtr.dev", "wss://relay.nostr.info", "wss://nostr-pub.wellorder.net" , "wss://relayer.fiatjaf.com", "wss://nodestr.fmt.wiz.biz/", "wss://no.str.cr" + ] relay_manager = RelayManager() def __init__(self, relays: List[str] = [], connect=True): From 6c66b71302c85198ff890db04ff2af60e9c2adac Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:23 +0300 Subject: [PATCH 44/96] chore: code clean-up --- nostr/relay_manager.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 0ec324a..004b197 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -79,14 +79,3 @@ class RelayManager: for relay in self.relays.values(): if relay.policy.should_write: relay.publish(message) - - def publish_event(self, event: Event): - """Verifies that the Event is publishable before submitting it to relays""" - if event.signature is None: - raise RelayException(f"Could not publish {event.id}: must be signed") - - if not event.verify(): - raise RelayException( - f"Could not publish {event.id}: failed to verify signature {event.signature}" - ) - self.publish_message(event.to_message()) From 62641b56a6c7d0283d938b92cd69b437addbcb65 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:42 +0300 Subject: [PATCH 45/96] chore: code clean-up --- views_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/views_api.py b/views_api.py index 88193ad..26a68e9 100644 --- a/views_api.py +++ b/views_api.py @@ -17,7 +17,6 @@ from .models import Relay, RelayList, TestMessage, TestMessageResponse from .nostr.key import EncryptedDirectMessage, PrivateKey from .nostr.relay import Relay as NostrRelay from .services import NostrRouter, nostr -from .tasks import init_relays # we keep this in all_routers: list[NostrRouter] = [] From 414ae16cb0cbad9d457730a16925d2984e9563a1 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:57 +0300 Subject: [PATCH 46/96] refactor: split large methods --- nostr/relay.py | 1 + services.py | 162 ++++++++++++++++++++++++++----------------------- 2 files changed, 86 insertions(+), 77 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 94e532c..8d3545b 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -121,6 +121,7 @@ class Relay: def close_subscription(self, id: str) -> None: with self.lock: self.subscriptions.pop(id) + self.publish(json.dumps(["CLOSE", id])) def to_json_object(self) -> dict: return { diff --git a/services.py b/services.py index b270539..42e8c2c 100644 --- a/services.py +++ b/services.py @@ -2,17 +2,16 @@ import asyncio import json from typing import List, Union -from fastapi import WebSocket, WebSocketDisconnect +from fastapi import WebSocketDisconnect from loguru import logger from lnbits.helpers import urlsafe_short_hash -from .models import Event, Filter, Filters, Relay, RelayList +from .models import Event, Filter from .nostr.client.client import NostrClient as NostrClientLib -from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters -from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage +from .nostr.message_pool import EndOfStoredEventsMessage, NoticeMessage received_subscription_events: dict[str, list[Event]] = {} received_subscription_notices: list[NoticeMessage] = [] @@ -33,7 +32,7 @@ class NostrRouter: self.connected: bool = True self.websocket = websocket self.tasks: List[asyncio.Task] = [] - self.oridinal_subscription_ids = {} + self.original_subscription_ids = {} async def client_to_nostr(self): """Receives requests / data from the client and forwards it to relays. If the @@ -47,17 +46,7 @@ class NostrRouter: self.connected = False break - # registers a subscription if the input was a REQ request - subscription_id, json_str_rewritten = await self._handle_nostr_subscription( - json_str - ) - - if subscription_id and json_str_rewritten: - self.subscriptions.append(subscription_id) - - # publish data - publish_data = json_str_rewritten or json_str - nostr.client.relay_manager.publish_message(publish_data) + await self._handle_client_to_nostr(json_str) async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -67,50 +56,12 @@ class NostrRouter: that we had previously rewritten in order to avoid collisions when multiple clients use the same id. """ while True and self.connected: - for s in self.subscriptions: - if s in received_subscription_events: - while len(received_subscription_events[s]): - my_event = received_subscription_events[s].pop(0) - # event.to_message() does not include the subscription ID, we have to add it manually - event_json = { - "id": my_event.id, - "pubkey": my_event.public_key, - "created_at": my_event.created_at, - "kind": my_event.kind, - "tags": my_event.tags, - "content": my_event.content, - "sig": my_event.signature, - } - - # this reconstructs the original response from the relay - # reconstruct original subscription id - s_original = self.oridinal_subscription_ids[s] - event_to_forward = ["EVENT", s_original, event_json] - - # print("Event to forward") - # print(json.dumps(event_to_forward)) - - # send data back to client - await self.websocket.send_text(json.dumps(event_to_forward)) - if s in received_subscription_eosenotices: - my_event = received_subscription_eosenotices[s] - s_original = self.oridinal_subscription_ids[s] - event_to_forward = ["EOSE", s_original] - del received_subscription_eosenotices[s] - # send data back to client - # print("Sending EOSE", event_to_forward) - await self.websocket.send_text(json.dumps(event_to_forward)) - - # if s in received_subscription_notices: - while len(received_subscription_notices): - my_event = received_subscription_notices.pop(0) - event_to_forward = ["NOTICE", my_event.content] - # send data back to client - logger.debug("Nostrclient: Received notice", event_to_forward[1]) - # note: we don't send it to the user because we don't know who should receive it - # await self.websocket.send_text(json.dumps(event_to_forward)) + await self._handle_subscriptions() + self._handle_notices() + await asyncio.sleep(0.1) + async def start(self): self.tasks.append(asyncio.create_task(self.client_to_nostr())) self.tasks.append(asyncio.create_task(self.nostr_to_client())) @@ -120,6 +71,53 @@ class NostrRouter: t.cancel() self.connected = False + async def _handle_subscriptions(self): + for s in self.subscriptions: + if s in received_subscription_events: + await self._handle_received_subscription_events(s) + if s in received_subscription_eosenotices: + await self._handle_received_subscription_eosenotices(s) + + + + async def _handle_received_subscription_eosenotices(self, s): + my_event = received_subscription_eosenotices[s] + s_original = self.original_subscription_ids[s] + event_to_forward = ["EOSE", s_original] + del received_subscription_eosenotices[s] + # send data back to client + # print("Sending EOSE", event_to_forward) + await self.websocket.send_text(json.dumps(event_to_forward)) + + async def _handle_received_subscription_events(self, s): + while len(received_subscription_events[s]): + my_event = received_subscription_events[s].pop(0) + # event.to_message() does not include the subscription ID, we have to add it manually + event_json = { + "id": my_event.id, + "pubkey": my_event.public_key, + "created_at": my_event.created_at, + "kind": my_event.kind, + "tags": my_event.tags, + "content": my_event.content, + "sig": my_event.signature, + } + + # this reconstructs the original response from the relay + # reconstruct original subscription id + s_original = self.original_subscription_ids[s] + event_to_forward = ["EVENT", s_original, event_json] + await self.websocket.send_text(json.dumps(event_to_forward)) + + def _handle_notices(self): + while len(received_subscription_notices): + my_event = received_subscription_notices.pop(0) + event_to_forward = ["NOTICE", my_event.content] + # note: we don't send it to the user because we don't know who should receive it + logger.debug("Nostrclient: Received notice", event_to_forward[1]) + + + def _marshall_nostr_filters(self, data: Union[dict, list]): filters = data if isinstance(data, list) else [data] filters = [Filter.parse_obj(f) for f in filters] @@ -139,7 +137,7 @@ class NostrRouter: ) return NostrFilters(filter_list) - async def _handle_nostr_subscription(self, json_str): + async def _handle_client_to_nostr(self, json_str): """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will register the subscription in the nostr client library that we're using so we can receive the callbacks on it later. Will rewrite the subscription id since we expect @@ -147,25 +145,35 @@ class NostrRouter: """ json_data = json.loads(json_str) assert len(json_data) + if json_data[0] == "REQ": - subscription_id = json_data[1] - subscription_id_rewritten = urlsafe_short_hash() - self.oridinal_subscription_ids[subscription_id_rewritten] = subscription_id - fltr = json_data[2] - filters = self._marshall_nostr_filters(fltr) - nostr.client.relay_manager.add_subscription( + self._handle_client_req(json_data) + return + + if json_data[0] == "CLOSE": + self.handle_client_close(json_data[1]) + return + + if json_data[0] == "EVENT": + nostr.client.relay_manager.publish_message(json_str) + return + + def _handle_client_req(self, json_data): + subscription_id = json_data[1] + subscription_id_rewritten = urlsafe_short_hash() + self.original_subscription_ids[subscription_id_rewritten] = subscription_id + fltr = json_data[2] + filters = self._marshall_nostr_filters(fltr) + + nostr.client.relay_manager.add_subscription( subscription_id_rewritten, filters ) - request_rewritten = json.dumps( - [json_data[0], subscription_id_rewritten, fltr] - ) - return subscription_id_rewritten, request_rewritten - elif json_data[0] == "CLOSE": - subscription_id = json_data[1] - subscription_id_rewritten = next((k for k, v in self.oridinal_subscription_ids.items() if v == subscription_id), None) - if subscription_id_rewritten: - nostr.client.relay_manager.close_subscription(subscription_id_rewritten) - request_rewritten = json.dumps([json_data[0], subscription_id_rewritten]) - return None, request_rewritten + request_rewritten = json.dumps([json_data[0], subscription_id_rewritten, fltr]) + + self.subscriptions.append(subscription_id_rewritten) + nostr.client.relay_manager.publish_message(request_rewritten) - return None, None + def handle_client_close(self, subscription_id): + subscription_id_rewritten = next((k for k, v in self.original_subscription_ids.items() if v == subscription_id), None) + if subscription_id_rewritten: + nostr.client.relay_manager.close_subscription(subscription_id_rewritten) From 4238be498f655693604fa1df79fdc6e9eab56a96 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:19:22 +0300 Subject: [PATCH 47/96] chore: code clean-up --- nostr/relay.py | 6 ------ nostr/relay_manager.py | 1 - services.py | 6 ++---- tasks.py | 3 --- views_api.py | 1 - 5 files changed, 2 insertions(+), 15 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 8d3545b..cc992f0 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -47,7 +47,6 @@ class Relay: self.queue = Queue() def connect(self, ssl_options: dict = None, proxy: dict = None): - print("### relay.connect", self.url) self.ws = WebSocketApp( self.url, on_open=self._on_open, @@ -84,7 +83,6 @@ class Relay: @property def ping(self): - print("### ping: ", self.url) ping_ms = int((self.ws.last_pong_tm - self.ws.last_ping_tm) * 1000) return ping_ms if self.connected and ping_ms > 0 else 0 @@ -98,21 +96,17 @@ class Relay: self.publish(json_str) def queue_worker(self): - print("#### IN !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) while True: if self.connected: try: message = self.queue.get(timeout=1) - print("#### queue_worker", message) self.num_sent_events += 1 self.ws.send(message) except Exception as e: if self.shutdown: - print("#### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! e [", e, self.url, self.shutdown," ]###") break else: time.sleep(0.1) - print("#### OUT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) def add_subscription(self, id, filters: Filters): with self.lock: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 004b197..b2e4fc2 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -2,7 +2,6 @@ import ssl import threading -from .event import Event from .filter import Filters from .message_pool import MessagePool from .relay import Relay, RelayPolicy diff --git a/services.py b/services.py index 42e8c2c..ed87b39 100644 --- a/services.py +++ b/services.py @@ -81,12 +81,10 @@ class NostrRouter: async def _handle_received_subscription_eosenotices(self, s): - my_event = received_subscription_eosenotices[s] s_original = self.original_subscription_ids[s] event_to_forward = ["EOSE", s_original] del received_subscription_eosenotices[s] - # send data back to client - # print("Sending EOSE", event_to_forward) + await self.websocket.send_text(json.dumps(event_to_forward)) async def _handle_received_subscription_events(self, s): @@ -114,7 +112,7 @@ class NostrRouter: my_event = received_subscription_notices.pop(0) event_to_forward = ["NOTICE", my_event.content] # note: we don't send it to the user because we don't know who should receive it - logger.debug("Nostrclient: Received notice", event_to_forward[1]) + logger.debug("Nostrclient: Received notice: ", event_to_forward[1]) diff --git a/tasks.py b/tasks.py index 069c57d..1db0d86 100644 --- a/tasks.py +++ b/tasks.py @@ -4,10 +4,7 @@ import ssl import threading from .crud import get_relays -from .nostr.event import Event -from .nostr.key import PublicKey from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage -from .nostr.relay_manager import RelayManager from .services import ( nostr, received_subscription_eosenotices, diff --git a/views_api.py b/views_api.py index 26a68e9..a16599b 100644 --- a/views_api.py +++ b/views_api.py @@ -1,5 +1,4 @@ import asyncio -import json from http import HTTPStatus from typing import List, Optional From c0632cabe53aa579cd9753fde8170a3a5bed0cbf Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 14:33:26 +0300 Subject: [PATCH 48/96] refactor: move stop logic --- services.py | 11 ++++++++++- views_api.py | 5 ----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/services.py b/services.py index ed87b39..5546d90 100644 --- a/services.py +++ b/services.py @@ -68,7 +68,16 @@ class NostrRouter: async def stop(self): for t in self.tasks: - t.cancel() + try: + t.cancel() + except: + pass + + for s in self.subscriptions: + try: + nostr.client.relay_manager.close_subscription(s) + except: + pass self.connected = False async def _handle_subscriptions(self): diff --git a/views_api.py b/views_api.py index a16599b..1a8d227 100644 --- a/views_api.py +++ b/views_api.py @@ -152,11 +152,6 @@ async def ws_relay(websocket: WebSocket) -> None: while True: await asyncio.sleep(10) if not router.connected: - for s in router.subscriptions: - try: - nostr.client.relay_manager.close_subscription(s) - except: - pass await router.stop() all_routers.remove(router) break From af14e1c47bb6afc426e711af86673f2eb9965c73 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 15:15:00 +0300 Subject: [PATCH 49/96] fix: do not lose subscriptions if no relay --- nostr/relay_manager.py | 9 +++++++++ views_api.py | 4 +--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index b2e4fc2..63eb68f 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,4 +1,5 @@ +from asyncio import Lock import ssl import threading @@ -18,6 +19,8 @@ class RelayManager: self.threads: dict[str, threading.Thread] = {} self.queue_threads: dict[str, threading.Thread] = {} self.message_pool = MessagePool() + self._cached_subscriptions = dict[str, Subscription] = {} + self._subscriptions_lock = Lock() def add_relay( self, url: str, read: bool = True, write: bool = True, subscriptions: dict[str, Subscription] = {} @@ -46,10 +49,16 @@ class RelayManager: self.relays.pop(url) def add_subscription(self, id: str, filters: Filters): + with self._subscriptions_lock: + self._cached_subscriptions[id] = Subscription(id, filters) + for relay in self.relays.values(): relay.add_subscription(id, filters) def close_subscription(self, id: str): + with self._subscriptions_lock: + self._cached_subscriptions.pop(id) + for relay in self.relays.values(): relay.close_subscription(id) diff --git a/views_api.py b/views_api.py index 1a8d227..0036070 100644 --- a/views_api.py +++ b/views_api.py @@ -62,11 +62,9 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: all_relays: List[NostrRelay] = nostr.client.relay_manager.relays.values() if len(all_relays): - subscriptions = all_relays[0].subscriptions nostr.client.relays.append(relay.url) - nostr.client.relay_manager.add_relay(subscriptions) + nostr.client.relay_manager.add_relay() - nostr.client.relay_manager.connect_relay(relay.url) return await get_relays() From 811bfdc45a707192d0c43ac91738ae76344cbc3f Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 16:55:09 +0300 Subject: [PATCH 50/96] fix: init new relays with previous subscriptions --- nostr/client/client.py | 4 ++-- nostr/relay_manager.py | 16 ++++++++-------- views_api.py | 6 ++---- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 0ff85a8..7c6063e 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -20,9 +20,9 @@ class NostrClient: if connect: self.connect() - async def connect(self, subscriptions: dict[str, Subscription] = {}): + async def connect(self): for relay in self.relays: - self.relay_manager.add_relay(relay, subscriptions) + self.relay_manager.add_relay(relay) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 63eb68f..ddc833c 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,5 +1,4 @@ -from asyncio import Lock import ssl import threading @@ -19,17 +18,18 @@ class RelayManager: self.threads: dict[str, threading.Thread] = {} self.queue_threads: dict[str, threading.Thread] = {} self.message_pool = MessagePool() - self._cached_subscriptions = dict[str, Subscription] = {} - self._subscriptions_lock = Lock() + self._cached_subscriptions: dict[str, Subscription] = {} + self._subscriptions_lock = threading.Lock() - def add_relay( - self, url: str, read: bool = True, write: bool = True, subscriptions: dict[str, Subscription] = {} - ) -> Relay: + def add_relay(self, url: str, read: bool = True, write: bool = True) -> Relay: if url in self.relays: return - + + with self._subscriptions_lock: + subscriptions = self._cached_subscriptions.copy() + policy = RelayPolicy(read, write) - relay = Relay(url, policy, self.message_pool, subscriptions.copy()) + relay = Relay(url, policy, self.message_pool, subscriptions) self.relays[url] = relay self.open_connection( diff --git a/views_api.py b/views_api.py index 0036070..131da44 100644 --- a/views_api.py +++ b/views_api.py @@ -60,10 +60,8 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: relay.id = urlsafe_short_hash() await add_relay(relay) - all_relays: List[NostrRelay] = nostr.client.relay_manager.relays.values() - if len(all_relays): - nostr.client.relays.append(relay.url) - nostr.client.relay_manager.add_relay() + nostr.client.relays.append(relay.url) + nostr.client.relay_manager.add_relay(relay.url) return await get_relays() From defb9b8963efe01f4386c472c4c7d7953ddfe09d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 17:10:34 +0300 Subject: [PATCH 51/96] chore: code clean-up --- nostr/client/client.py | 9 +-------- tasks.py | 2 -- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 7c6063e..adc1a6b 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -6,12 +6,7 @@ from ..subscription import Subscription class NostrClient: - relays = [ - "wss://nostr-pub.wellorder.net", - "wss://nostr.zebedee.cloud", - "wss://nodestr.fmt.wiz.biz", - "wss://nostr.oxtr.dev", - ] + relays = [ ] relay_manager = RelayManager() def __init__(self, relays: List[str] = [], connect=True): @@ -24,8 +19,6 @@ class NostrClient: for relay in self.relays: self.relay_manager.add_relay(relay) - - def close(self): self.relay_manager.close_connections() diff --git a/tasks.py b/tasks.py index 1db0d86..7d471fc 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,4 @@ import asyncio -import json -import ssl import threading from .crud import get_relays From 64426d187c31c1ae50f2b020b0d53a5a0e403c09 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 17:26:47 +0300 Subject: [PATCH 52/96] feat: add predefined relay list plus code format --- templates/nostrclient/index.html | 191 +++++++++---------------------- 1 file changed, 53 insertions(+), 138 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 10fd4a5..7e83b6d 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -6,18 +6,18 @@
- +
- Add relay - + + + + + + + +
@@ -29,36 +29,18 @@
Nostrclient
- +
- + @@ -255,6 +256,13 @@ label: 'Ping', field: 'ping' } + , + { + name: 'delete', + align: 'center', + label: '', + field: '' + } ], pagination: { rowsPerPage: 10 @@ -329,8 +337,14 @@ this.relayToAdd = relayUrl await this.addRelay() }, + showDeleteRelayDialog: function (url) { + LNbits.utils + .confirmDialog(' Are you sure you want to remove this relay?') + .onOk(async () => { + this.deleteRelay(url) + }) + }, deleteRelay(url) { - console.log('DELETE RELAY ' + url) LNbits.api .request( 'DELETE', @@ -338,12 +352,14 @@ this.g.user.wallets[0].adminkey, { url: url } ) - .then(function (response) { - if (response.data) { - console.log(response.data) + .then((response) => { + const relayIndex = this.nostrrelayLinks.indexOf(r => r.url === url) + if (relayIndex !== -1) { + this.nostrrelayLinks.splice(relayIndex, 1) } }) - .catch(function (error) { + .catch((error) => { + console.error(error) LNbits.utils.notifyApiError(error) }) }, From ce8b95c2c7a700885ff533be126bd37876ccf8d7 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 10:21:06 +0300 Subject: [PATCH 60/96] feat: restart disconnected relays --- __init__.py | 4 +++- nostr/relay.py | 44 +++++++++++++++++++++--------------------- nostr/relay_manager.py | 37 ++++++++++++++++++++++++----------- tasks.py | 12 ++++++++++++ 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/__init__.py b/__init__.py index e98d8b8..ec6f4b4 100644 --- a/__init__.py +++ b/__init__.py @@ -36,7 +36,7 @@ def nostr_renderer(): return template_renderer(["lnbits/extensions/nostrclient/templates"]) -from .tasks import init_relays, subscribe_events +from .tasks import check_relays, init_relays, subscribe_events from .views import * # noqa from .views_api import * # noqa @@ -47,3 +47,5 @@ def nostrclient_start(): scheduled_tasks.append(task1) task2 = loop.create_task(catch_everything_and_restart(subscribe_events)) scheduled_tasks.append(task2) + task3 = loop.create_task(catch_everything_and_restart(check_relays)) + scheduled_tasks.append(task3) diff --git a/nostr/relay.py b/nostr/relay.py index cc992f0..6ff16f5 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -3,6 +3,7 @@ import time from queue import Queue from threading import Lock +from loguru import logger from websocket import WebSocketApp from .event import Event @@ -69,17 +70,12 @@ class Relay: def close(self): self.ws.close() + self.connected = False self.shutdown = True - def check_reconnect(self): - try: - self.close() - except: - pass - self.connected = False - if self.reconnect: - time.sleep(self.error_counter**2) - self.connect(self.ssl_options, self.proxy) + @property + def error_threshold_reached(self): + return self.error_threshold and self.error_counter > self.error_threshold @property def ping(self): @@ -104,6 +100,7 @@ class Relay: self.ws.send(message) except Exception as e: if self.shutdown: + logger.warning(f"Closing queue worker for {self.url}") break else: time.sleep(0.1) @@ -127,31 +124,34 @@ class Relay: ], } - def _on_open(self, class_obj): + def _on_open(self, _): + logger.info(f"Connected to relay: '{self.url}'.") self.connected = True - pass + - def _on_close(self, class_obj, status_code, message): - self.connected = False - if self.error_threshold and self.error_counter > self.error_threshold: - pass - else: - self.check_reconnect() - pass + def _on_close(self, _, status_code, message): + logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") + self.close() - def _on_message(self, class_obj, message: str): + + + + def _on_message(self, _, message: str): if self._is_valid_message(message): self.num_received_events += 1 self.message_pool.add_message(message, self.url) + else: + logger.debug(f"Invalid relay message: '{message}'.") - def _on_error(self, class_obj, error): + def _on_error(self, _, error): + logger.warning(f"Relay error: '{str(error)}'") self.connected = False self.error_counter += 1 - def _on_ping(self, class_obj, message): + def _on_ping(self, _*): return - def _on_pong(self, class_obj, message): + def _on_pong(self, _*): return def _is_valid_message(self, message: str) -> bool: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index ddc833c..3ccc2fd 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -2,6 +2,8 @@ import ssl import threading +from loguru import logger + from .filter import Filters from .message_pool import MessagePool from .relay import Relay, RelayPolicy @@ -22,7 +24,7 @@ class RelayManager: self._subscriptions_lock = threading.Lock() def add_relay(self, url: str, read: bool = True, write: bool = True) -> Relay: - if url in self.relays: + if url in list(self.relays.keys()): return with self._subscriptions_lock: @@ -32,7 +34,7 @@ class RelayManager: relay = Relay(url, policy, self.message_pool, subscriptions) self.relays[url] = relay - self.open_connection( + self._open_connection( relay, {"cert_reqs": ssl.CERT_NONE} ) # NOTE: This disables ssl certificate verification @@ -62,8 +64,23 @@ class RelayManager: for relay in self.relays.values(): relay.close_subscription(id) + def check_and_restart_relays(self): + stopped_relays = [r for r in self.relays.values() if r.shutdown] + for relay in stopped_relays: + logger.info(f"Restarting connection to relay '{relay.url}'") + self._restart_relay(relay) - def open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): + + def close_connections(self): + for relay in self.relays.values(): + relay.close() + + def publish_message(self, message: str): + for relay in self.relays.values(): + if relay.policy.should_write: + relay.publish(message) + + def _open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): self.threads[relay.url] = threading.Thread( target=relay.connect, args=(ssl_options, proxy), @@ -79,11 +96,9 @@ class RelayManager: ) self.queue_threads[relay.url].start() - def close_connections(self): - for relay in self.relays.values(): - relay.close() - - def publish_message(self, message: str): - for relay in self.relays.values(): - if relay.policy.should_write: - relay.publish(message) + def _restart_relay(self, relay: Relay): + if relay.error_threshold_reached: + return + self.remove_relay(relay.url) + new_relay = self.add_relay(relay.url) + new_relay.error_counter = relay.error_counter \ No newline at end of file diff --git a/tasks.py b/tasks.py index 40ca9d9..05057e7 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,8 @@ import asyncio import threading +from loguru import logger + from . import nostr from .crud import get_relays from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage @@ -17,6 +19,16 @@ async def init_relays(): await nostr.client.connect() +async def check_relays(): + """ Check relays that have been disconnected """ + while True: + try: + await asyncio.sleep(20) + nostr.client.relay_manager.check_and_restart_relays() + except Exception as e: + logger.warning(f"Cannot restart relays: '{str(e)}'.") + + async def subscribe_events(): while not any([r.connected for r in nostr.client.relay_manager.relays.values()]): await asyncio.sleep(2) From dabc26f8a6eab854efe5c3e56e8a6a3fb62612e5 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 10:53:48 +0300 Subject: [PATCH 61/96] fix: unused params --- nostr/relay.py | 4 ++-- nostr/relay_manager.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 6ff16f5..d98a219 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -148,10 +148,10 @@ class Relay: self.connected = False self.error_counter += 1 - def _on_ping(self, _*): + def _on_ping(self, *_): return - def _on_pong(self, _*): + def _on_pong(self, *_): return def _is_valid_message(self, message: str) -> bool: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 3ccc2fd..01889d9 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -67,7 +67,6 @@ class RelayManager: def check_and_restart_relays(self): stopped_relays = [r for r in self.relays.values() if r.shutdown] for relay in stopped_relays: - logger.info(f"Restarting connection to relay '{relay.url}'") self._restart_relay(relay) @@ -99,6 +98,8 @@ class RelayManager: def _restart_relay(self, relay: Relay): if relay.error_threshold_reached: return + logger.info(f"Restarting connection to relay '{relay.url}'") + self.remove_relay(relay.url) new_relay = self.add_relay(relay.url) new_relay.error_counter = relay.error_counter \ No newline at end of file From f8d578e6aa9f3d3d881cf0fd52d23e18aa553a31 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 12:20:06 +0300 Subject: [PATCH 62/96] feat: improve error handling and reporting --- models.py | 8 ++- nostr/message_type.py | 11 ++++- nostr/relay.py | 84 ++++++++++++++++++++------------ nostr/relay_manager.py | 3 +- templates/nostrclient/index.html | 19 ++++++++ views_api.py | 18 ++++--- 6 files changed, 101 insertions(+), 42 deletions(-) diff --git a/models.py b/models.py index fe12e3b..1006605 100644 --- a/models.py +++ b/models.py @@ -8,12 +8,18 @@ from pydantic import BaseModel, Field from lnbits.helpers import urlsafe_short_hash +class RelayStatus(BaseModel): + num_sent_events: Optional[int] = 0 + num_received_events: Optional[int] = 0 + error_counter: Optional[int] = 0 + error_list: Optional[List] = [] + class Relay(BaseModel): id: Optional[str] = None url: Optional[str] = None connected: Optional[bool] = None connected_string: Optional[str] = None - status: Optional[str] = None + status: Optional[RelayStatus] = None active: Optional[bool] = None ping: Optional[int] = None diff --git a/nostr/message_type.py b/nostr/message_type.py index 3f5206b..d37cdfd 100644 --- a/nostr/message_type.py +++ b/nostr/message_type.py @@ -3,13 +3,20 @@ class ClientMessageType: REQUEST = "REQ" CLOSE = "CLOSE" + class RelayMessageType: EVENT = "EVENT" NOTICE = "NOTICE" END_OF_STORED_EVENTS = "EOSE" + COMMAND_RESULT = "OK" @staticmethod def is_valid(type: str) -> bool: - if type == RelayMessageType.EVENT or type == RelayMessageType.NOTICE or type == RelayMessageType.END_OF_STORED_EVENTS: + if ( + type == RelayMessageType.EVENT + or type == RelayMessageType.NOTICE + or type == RelayMessageType.END_OF_STORED_EVENTS + or type == RelayMessageType.COMMAND_RESULT + ): return True - return False \ No newline at end of file + return False diff --git a/nostr/relay.py b/nostr/relay.py index d98a219..4c989c2 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,6 +2,7 @@ import json import time from queue import Queue from threading import Lock +from typing import List from loguru import logger from websocket import WebSocketApp @@ -39,6 +40,7 @@ class Relay: self.shutdown: bool = False self.error_counter: int = 0 self.error_threshold: int = 100 + self.error_list: List[str] = [] self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 @@ -100,7 +102,7 @@ class Relay: self.ws.send(message) except Exception as e: if self.shutdown: - logger.warning(f"Closing queue worker for {self.url}") + logger.warning(f"Closing queue worker for '{self.url}'.") break else: time.sleep(0.1) @@ -133,18 +135,14 @@ class Relay: logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") self.close() - - - def _on_message(self, _, message: str): if self._is_valid_message(message): self.num_received_events += 1 self.message_pool.add_message(message, self.url) - else: - logger.debug(f"Invalid relay message: '{message}'.") def _on_error(self, _, error): logger.warning(f"Relay error: '{str(error)}'") + self._append_error_message(str(error)) self.connected = False self.error_counter += 1 @@ -161,33 +159,57 @@ class Relay: message_json = json.loads(message) message_type = message_json[0] + if not RelayMessageType.is_valid(message_type): return False + if message_type == RelayMessageType.EVENT: - if not len(message_json) == 3: - return False - - subscription_id = message_json[1] - with self.lock: - if subscription_id not in self.subscriptions: - return False - - e = message_json[2] - event = Event( - e["content"], - e["pubkey"], - e["created_at"], - e["kind"], - e["tags"], - e["sig"], - ) - if not event.verify(): - return False - - with self.lock: - subscription = self.subscriptions[subscription_id] - - if subscription.filters and not subscription.filters.match(event): - return False + return self._is_valid_event_message(message_json) + + if message_type == RelayMessageType.COMMAND_RESULT: + return self._is_valid_command_result_message(message, message_json) return True + + def _is_valid_event_message(self, message_json): + if not len(message_json) == 3: + return False + + subscription_id = message_json[1] + with self.lock: + if subscription_id not in self.subscriptions: + return False + + e = message_json[2] + event = Event( + e["content"], + e["pubkey"], + e["created_at"], + e["kind"], + e["tags"], + e["sig"], + ) + if not event.verify(): + return False + + with self.lock: + subscription = self.subscriptions[subscription_id] + + if subscription.filters and not subscription.filters.match(event): + return False + + return True + + def _is_valid_command_result_message(self, message, message_json): + if not len(message_json) < 3: + return False + + if message_json[2] != True: + logger.warning(f"Relay '{self.url}' negative command result: '{message}'") + self._append_error_message(message) + return False + + return True + + def _append_error_message(self, message): + self.error_list = ([message] + self.error_list)[:20] \ No newline at end of file diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 01889d9..5838308 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -102,4 +102,5 @@ class RelayManager: self.remove_relay(relay.url) new_relay = self.add_relay(relay.url) - new_relay.error_counter = relay.error_counter \ No newline at end of file + new_relay.error_counter = relay.error_counter + new_relay.error_list = relay.error_list \ No newline at end of file diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index f200973..8018a92 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -51,6 +51,18 @@
+
+
+ ⬆️ + ⬇️ + ⚠️ + + + +
+ +
+
{{ col.value }} @@ -196,6 +208,13 @@ obj._data = _.clone(obj) obj.theTime = obj.time * 60 - (Date.now() / 1000 - obj.timestamp) obj.time = obj.time + 'mins' + obj.status = { + sentEvents: obj.status.num_sent_events, + receveidEvents: obj.status.num_received_events, + errorCount: obj.status.error_counter, + errorList: obj.status.error_list + + } obj.ping = obj.ping + ' ms' diff --git a/views_api.py b/views_api.py index 316ec4d..f918e3c 100644 --- a/views_api.py +++ b/views_api.py @@ -24,19 +24,23 @@ all_routers: list[NostrRouter] = [] async def api_get_relays() -> RelayList: relays = RelayList(__root__=[]) for url, r in nostr.client.relay_manager.relays.items(): - status_text = ( - f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" - ) - connected_text = "🟢" if r.connected else "🔴" + # status_text = ( + # f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" + # ) + # connected_text = "🟢" if r.connected else "🔴" relay_id = urlsafe_short_hash() relays.__root__.append( Relay( id=relay_id, url=url, - connected_string=connected_text, - status=status_text, + connected=r.connected, + status={ + "num_sent_events": r.num_sent_events, + "num_received_events": r.num_received_events, + "error_counter": r.error_counter, + "error_list": r.error_list + }, ping=r.ping, - connected=True, active=True, ) ) From d619b965e71e59143331b523482add58deda684f Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 12:59:58 +0300 Subject: [PATCH 63/96] fix: UI for connected --- templates/nostrclient/index.html | 36 ++++++++++++++++---------------- views_api.py | 4 ---- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 8018a92..57c3b73 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -50,30 +50,30 @@ @@ -252,10 +252,10 @@ relayTable: { columns: [ { - name: 'connected_string', + name: 'connected', align: 'left', label: '', - field: 'connected_string' + field: 'connected' }, { name: 'relay', diff --git a/views_api.py b/views_api.py index f918e3c..b681f50 100644 --- a/views_api.py +++ b/views_api.py @@ -24,10 +24,6 @@ all_routers: list[NostrRouter] = [] async def api_get_relays() -> RelayList: relays = RelayList(__root__=[]) for url, r in nostr.client.relay_manager.relays.items(): - # status_text = ( - # f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" - # ) - # connected_text = "🟢" if r.connected else "🔴" relay_id = urlsafe_short_hash() relays.__root__.append( Relay( From ada5b2a51d4c805854adab568fd939edb0e57572 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 29 Jun 2023 15:48:28 +0300 Subject: [PATCH 64/96] fix: unsubscribe --- router.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/router.py b/router.py index 3ee9d0a..264d4a4 100644 --- a/router.py +++ b/router.py @@ -39,7 +39,11 @@ class NostrRouter: self.connected = False break - await self._handle_client_to_nostr(json_str) + try: + await self._handle_client_to_nostr(json_str) + except Exception as e: + logger.debug(f"Failed to handle client message: '{str(e)}'.") + async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -49,10 +53,13 @@ class NostrRouter: that we had previously rewritten in order to avoid collisions when multiple clients use the same id. """ while True and self.connected: - await self._handle_subscriptions() - self._handle_notices() - - await asyncio.sleep(0.1) + try: + await self._handle_subscriptions() + self._handle_notices() + await asyncio.sleep(0.1) + except Exception as e: + logger.debug(f"Failed to handle response for client: '{str(e)}'.") + async def start(self): @@ -112,9 +119,8 @@ class NostrRouter: def _handle_notices(self): while len(NostrRouter.received_subscription_notices): my_event = NostrRouter.received_subscription_notices.pop(0) - event_to_forward = ["NOTICE", my_event.content] # note: we don't send it to the user because we don't know who should receive it - logger.debug("Nostrclient: Received notice: ", event_to_forward[1]) + logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") @@ -143,9 +149,11 @@ class NostrRouter: receive the callbacks on it later. Will rewrite the subscription id since we expect multiple clients to use the router and want to avoid subscription id collisions """ + json_data = json.loads(json_str) assert len(json_data) + if json_data[0] == "REQ": self._handle_client_req(json_data) return @@ -176,4 +184,7 @@ class NostrRouter: def _handle_client_close(self, subscription_id): subscription_id_rewritten = next((k for k, v in self.original_subscription_ids.items() if v == subscription_id), None) if subscription_id_rewritten: + self.original_subscription_ids.pop(subscription_id_rewritten) nostr.client.relay_manager.close_subscription(subscription_id_rewritten) + else: + logger.debug(f"Failed to unsubscribe from '{subscription_id}.'") From 39c2f881c85c1337f5634106e82db72f73f65bf9 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 10:52:16 +0300 Subject: [PATCH 65/96] feat: revive relay after 24 hours from the last error --- nostr/relay.py | 7 +++++-- nostr/relay_manager.py | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 4c989c2..da3496d 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -41,6 +41,8 @@ class Relay: self.error_counter: int = 0 self.error_threshold: int = 100 self.error_list: List[str] = [] + self.notice_list: List[str] = [] + self.last_error_date: int = 0 self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 @@ -77,7 +79,7 @@ class Relay: @property def error_threshold_reached(self): - return self.error_threshold and self.error_counter > self.error_threshold + return self.error_threshold and self.error_counter >= self.error_threshold @property def ping(self): @@ -212,4 +214,5 @@ class Relay: return True def _append_error_message(self, message): - self.error_list = ([message] + self.error_list)[:20] \ No newline at end of file + self.error_list = ([message] + self.error_list)[:20] + self.last_error_date = int(time.time()) \ No newline at end of file diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 5838308..f8f852c 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,6 +1,7 @@ import ssl import threading +import time from loguru import logger @@ -97,7 +98,12 @@ class RelayManager: def _restart_relay(self, relay: Relay): if relay.error_threshold_reached: - return + time_since_last_error = time.time() - relay.last_error_date + if time_since_last_error < 60 * 60 * 24: # last day + return + relay.error_counter = 0 + relay.error_list = [] + logger.info(f"Restarting connection to relay '{relay.url}'") self.remove_relay(relay.url) From f244f60c562f8747e8b9dba91440e177a962271d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 10:53:39 +0300 Subject: [PATCH 66/96] fix: make it 2 hours --- nostr/relay_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index f8f852c..da650d6 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -99,7 +99,7 @@ class RelayManager: def _restart_relay(self, relay: Relay): if relay.error_threshold_reached: time_since_last_error = time.time() - relay.last_error_date - if time_since_last_error < 60 * 60 * 24: # last day + if time_since_last_error < 60 * 60 * 2: # last day return relay.error_counter = 0 relay.error_list = [] From 1601f71b03ffc2b53e6630f00495d83d50d41583 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 11:46:47 +0300 Subject: [PATCH 67/96] feat: show relay NOTICE --- models.py | 1 + nostr/relay.py | 4 +++- nostr/relay_manager.py | 7 +++++- router.py | 1 + templates/nostrclient/index.html | 38 ++++++++++++++++++++++++++------ views_api.py | 3 ++- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/models.py b/models.py index 1006605..88651fc 100644 --- a/models.py +++ b/models.py @@ -13,6 +13,7 @@ class RelayStatus(BaseModel): num_received_events: Optional[int] = 0 error_counter: Optional[int] = 0 error_list: Optional[List] = [] + notice_list: Optional[List] = [] class Relay(BaseModel): id: Optional[str] = None diff --git a/nostr/relay.py b/nostr/relay.py index da3496d..8b081a3 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -128,11 +128,13 @@ class Relay: ], } + def add_notice(self, notice: str): + self.notice_list = ([notice] + self.notice_list)[:20] + def _on_open(self, _): logger.info(f"Connected to relay: '{self.url}'.") self.connected = True - def _on_close(self, _, status_code, message): logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") self.close() diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index da650d6..b2df735 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -6,7 +6,7 @@ import time from loguru import logger from .filter import Filters -from .message_pool import MessagePool +from .message_pool import MessagePool, NoticeMessage from .relay import Relay, RelayPolicy from .subscription import Subscription @@ -80,6 +80,11 @@ class RelayManager: if relay.policy.should_write: relay.publish(message) + def handle_notice(self, notice: NoticeMessage): + relay = next((r for r in self.relays.values() if r.url == notice.url)) + if relay: + relay.add_notice(notice.content) + def _open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): self.threads[relay.url] = threading.Thread( target=relay.connect, diff --git a/router.py b/router.py index 264d4a4..0982f76 100644 --- a/router.py +++ b/router.py @@ -121,6 +121,7 @@ class NostrRouter: my_event = NostrRouter.received_subscription_notices.pop(0) # note: we don't send it to the user because we don't know who should receive it logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") + nostr.client.relay_manager.handle_notice(my_event) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 57c3b73..82b149e 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -58,12 +58,15 @@
⬆️ ⬇️ - ⚠️ - + + ⚠️ + - -
- + + ⓘ + + +
@@ -198,6 +201,17 @@
+ + + + + +
+ Close +
+ + +
{% endraw %} {% endblock %} {% block scripts %} {{ window_vars(user) }} @@ -212,8 +226,8 @@ sentEvents: obj.status.num_sent_events, receveidEvents: obj.status.num_received_events, errorCount: obj.status.error_counter, - errorList: obj.status.error_list - + errorList: obj.status.error_list, + noticeList: obj.status.notice_list } obj.ping = obj.ping + ' ms' @@ -226,6 +240,7 @@ 'HH:mm:ss' ) } + console.log('### obj', obj) return obj } @@ -239,6 +254,10 @@ relayToAdd: '', nostrrelayLinks: [], filter: '', + logData: { + show: false, + data: null + }, testData: { show: false, wsConnection: null, @@ -492,6 +511,11 @@ console.warn(error) } }, + showLogDataDialog: function (data = []) { + console.log('### showLogDataDialog', data) + this.logData.data = data.join('\n') + this.logData.show = true + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) diff --git a/views_api.py b/views_api.py index b681f50..b6b4527 100644 --- a/views_api.py +++ b/views_api.py @@ -34,7 +34,8 @@ async def api_get_relays() -> RelayList: "num_sent_events": r.num_sent_events, "num_received_events": r.num_received_events, "error_counter": r.error_counter, - "error_list": r.error_list + "error_list": r.error_list, + "notice_list": r.notice_list, }, ping=r.ping, active=True, From 80b86bf00c751cbcca11701bd9c3ab4ac9f2bd94 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 12:05:28 +0300 Subject: [PATCH 68/96] fix: await even if error --- router.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/router.py b/router.py index 0982f76..e85653c 100644 --- a/router.py +++ b/router.py @@ -56,10 +56,9 @@ class NostrRouter: try: await self._handle_subscriptions() self._handle_notices() - await asyncio.sleep(0.1) except Exception as e: logger.debug(f"Failed to handle response for client: '{str(e)}'.") - + await asyncio.sleep(0.1) async def start(self): From 147af04c20505aa42b21acb9903bf4193a7b504c Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 4 Jul 2023 10:02:47 +0300 Subject: [PATCH 69/96] fix: remove unnecessary `async` --- nostr/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index e033262..66d722c 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -14,7 +14,7 @@ class NostrClient: if connect: self.connect() - async def connect(self): + def connect(self): for relay in self.relays: self.relay_manager.add_relay(relay) From 3b08714a8472e4113110efb4a68095dd2b78cba6 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 4 Jul 2023 11:48:50 +0300 Subject: [PATCH 70/96] fix: async aaaa --- nostr/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 66d722c..e033262 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -14,7 +14,7 @@ class NostrClient: if connect: self.connect() - def connect(self): + async def connect(self): for relay in self.relays: self.relay_manager.add_relay(relay) From 403c8f1b05caf595df092cec7b42831a28ddc089 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 5 Jul 2023 09:09:10 +0300 Subject: [PATCH 71/96] fix: thread not dead --- nostr/relay.py | 10 ++++++---- nostr/relay_manager.py | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 8b081a3..0583bba 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -102,12 +102,14 @@ class Relay: message = self.queue.get(timeout=1) self.num_sent_events += 1 self.ws.send(message) - except Exception as e: - if self.shutdown: - logger.warning(f"Closing queue worker for '{self.url}'.") - break + except: + pass else: time.sleep(0.1) + + if self.shutdown: + logger.warning(f"Closing queue worker for '{self.url}'.") + break def add_subscription(self, id, filters: Filters): with self.lock: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index b2df735..a551253 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -44,12 +44,13 @@ class RelayManager: return relay def remove_relay(self, url: str): - self.threads[url].join(timeout=1) - self.threads.pop(url) - self.queue_threads[url].join(timeout=1) - self.queue_threads.pop(url) self.relays[url].close() self.relays.pop(url) + self.threads[url].join(timeout=5) + self.threads.pop(url) + self.queue_threads[url].join(timeout=5) + self.queue_threads.pop(url) + def add_subscription(self, id: str, filters: Filters): with self._subscriptions_lock: From 9d9fbc01895ec7b940b8e29a2fba850b5905755b Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 5 Jul 2023 09:09:20 +0300 Subject: [PATCH 72/96] chore: code clean-up --- templates/nostrclient/index.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 82b149e..a0c5999 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -240,7 +240,6 @@ 'HH:mm:ss' ) } - console.log('### obj', obj) return obj } @@ -512,7 +511,6 @@ } }, showLogDataDialog: function (data = []) { - console.log('### showLogDataDialog', data) this.logData.data = data.join('\n') this.logData.show = true }, From e6624f76bd0ead9d9f1e2259cfe40d3da7e75622 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 12 Sep 2023 15:06:28 +0300 Subject: [PATCH 73/96] Performance improvements (#19) * fix: increase the wait time for re-connecting to a relay * fix: blocking sleep * fix: remove blocking sleep * fix: allow multiple filters per request --- nostr/client/client.py | 6 +++--- nostr/relay.py | 5 +++-- nostr/relay_manager.py | 17 ++++++++++------- router.py | 4 ++-- tasks.py | 10 ++++++---- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index e033262..db07a06 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,4 +1,4 @@ -import time +import asyncio from typing import List from ..relay_manager import RelayManager @@ -21,7 +21,7 @@ class NostrClient: def close(self): self.relay_manager.close_connections() - def subscribe( + async def subscribe( self, callback_events_func=None, callback_notices_func=None, @@ -41,4 +41,4 @@ class NostrClient: if callback_eosenotices_func: callback_eosenotices_func(event_msg) - time.sleep(0.1) + await asyncio.sleep(0.5) diff --git a/nostr/relay.py b/nostr/relay.py index 0583bba..caacba0 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -1,3 +1,4 @@ +import asyncio import json import time from queue import Queue @@ -95,7 +96,7 @@ class Relay: json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) self.publish(json_str) - def queue_worker(self): + async def queue_worker(self): while True: if self.connected: try: @@ -105,7 +106,7 @@ class Relay: except: pass else: - time.sleep(0.1) + await asyncio.sleep(1) if self.shutdown: logger.warning(f"Closing queue worker for '{self.url}'.") diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index a551253..f639fb0 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,4 +1,5 @@ +import asyncio import ssl import threading import time @@ -95,20 +96,22 @@ class RelayManager: ) self.threads[relay.url].start() + def wrap_async_queue_worker(): + asyncio.run(relay.queue_worker()) + self.queue_threads[relay.url] = threading.Thread( - target=relay.queue_worker, + target=wrap_async_queue_worker, name=f"{relay.url}-queue", daemon=True, ) self.queue_threads[relay.url].start() def _restart_relay(self, relay: Relay): - if relay.error_threshold_reached: - time_since_last_error = time.time() - relay.last_error_date - if time_since_last_error < 60 * 60 * 2: # last day - return - relay.error_counter = 0 - relay.error_list = [] + time_since_last_error = time.time() - relay.last_error_date + + min_wait_time = min(60 * relay.error_counter, 60 * 60 * 24) # try at least once a day + if time_since_last_error < min_wait_time: + return logger.info(f"Restarting connection to relay '{relay.url}'") diff --git a/router.py b/router.py index e85653c..86a8f41 100644 --- a/router.py +++ b/router.py @@ -170,13 +170,13 @@ class NostrRouter: subscription_id = json_data[1] subscription_id_rewritten = urlsafe_short_hash() self.original_subscription_ids[subscription_id_rewritten] = subscription_id - fltr = json_data[2] + fltr = json_data[2:] filters = self._marshall_nostr_filters(fltr) nostr.client.relay_manager.add_subscription( subscription_id_rewritten, filters ) - request_rewritten = json.dumps([json_data[0], subscription_id_rewritten, fltr]) + request_rewritten = json.dumps([json_data[0], subscription_id_rewritten] + fltr) self.subscriptions.append(subscription_id_rewritten) nostr.client.relay_manager.publish_message(request_rewritten) diff --git a/tasks.py b/tasks.py index 05057e7..4c316bc 100644 --- a/tasks.py +++ b/tasks.py @@ -66,13 +66,15 @@ async def subscribe_events(): return - t = threading.Thread( - target=nostr.client.subscribe, - args=( + def wrap_async_subscribe(): + asyncio.run(nostr.client.subscribe( callback_events, callback_notices, callback_eose_notices, - ), + )) + + t = threading.Thread( + target=wrap_async_subscribe, name="Nostr-event-subscription", daemon=True, ) From 6f5e9e34589044225b372eed01594f01a17fc24f Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 15 Sep 2023 10:21:38 +0300 Subject: [PATCH 74/96] chore: doc formatting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1de61ba..02d12b7 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # Nostrclient - [LNbits](https://github.com/lnbits/lnbits) extension + For more about LNBits extension check [this tutorial](https://github.com/lnbits/lnbits/wiki/LNbits-Extensions) `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) - ### Troubleshoot -The `Test Endpoint` functionality heps the user to check that the `nostrclient` web-socket endpoint works as expected. + +The `Test Endpoint` functionality heps the user to check that the `nostrclient` web-socket endpoint works as expected. The LNbits user can DM itself (or a temp account) from `nostrclient` and verify that the messages are sent and received correctly. https://user-images.githubusercontent.com/2951406/236780745-929c33c2-2502-49be-84a3-db02a7aabc0e.mp4 - From e841183c1022eb50646802f4a907febc6ac691f3 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 25 Sep 2023 10:13:55 +0300 Subject: [PATCH 75/96] fix: add extra checks (#21) * fix: add extra checks * fix: remove redundant try-catch --- router.py | 56 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/router.py b/router.py index 86a8f41..cc0a380 100644 --- a/router.py +++ b/router.py @@ -89,31 +89,41 @@ class NostrRouter: async def _handle_received_subscription_eosenotices(self, s): - s_original = self.original_subscription_ids[s] - event_to_forward = ["EOSE", s_original] - del NostrRouter.received_subscription_eosenotices[s] - - await self.websocket.send_text(json.dumps(event_to_forward)) + try: + if s not in self.original_subscription_ids: + return + s_original = self.original_subscription_ids[s] + event_to_forward = ["EOSE", s_original] + del NostrRouter.received_subscription_eosenotices[s] + + await self.websocket.send_text(json.dumps(event_to_forward)) + except Exception as e: + logger.debug(e) async def _handle_received_subscription_events(self, s): - while len(NostrRouter.received_subscription_events[s]): - my_event = NostrRouter.received_subscription_events[s].pop(0) - # event.to_message() does not include the subscription ID, we have to add it manually - event_json = { - "id": my_event.id, - "pubkey": my_event.public_key, - "created_at": my_event.created_at, - "kind": my_event.kind, - "tags": my_event.tags, - "content": my_event.content, - "sig": my_event.signature, - } + try: + if s not in NostrRouter.received_subscription_events: + return + while len(NostrRouter.received_subscription_events[s]): + my_event = NostrRouter.received_subscription_events[s].pop(0) + # event.to_message() does not include the subscription ID, we have to add it manually + event_json = { + "id": my_event.id, + "pubkey": my_event.public_key, + "created_at": my_event.created_at, + "kind": my_event.kind, + "tags": my_event.tags, + "content": my_event.content, + "sig": my_event.signature, + } - # this reconstructs the original response from the relay - # reconstruct original subscription id - s_original = self.original_subscription_ids[s] - event_to_forward = ["EVENT", s_original, event_json] - await self.websocket.send_text(json.dumps(event_to_forward)) + # this reconstructs the original response from the relay + # reconstruct original subscription id + s_original = self.original_subscription_ids[s] + event_to_forward = ["EVENT", s_original, event_json] + await self.websocket.send_text(json.dumps(event_to_forward)) + except Exception as e: + logger.debug(e) def _handle_notices(self): while len(NostrRouter.received_subscription_notices): @@ -121,7 +131,7 @@ class NostrRouter: # note: we don't send it to the user because we don't know who should receive it logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") nostr.client.relay_manager.handle_notice(my_event) - + def _marshall_nostr_filters(self, data: Union[dict, list]): From d202fe305510f2714c9fb4b1ed2c83869c9e9200 Mon Sep 17 00:00:00 2001 From: Tiago Vasconcelos Date: Tue, 26 Sep 2023 13:41:58 +0100 Subject: [PATCH 76/96] allow custom path (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * allow custom path --------- Co-authored-by: dni ⚡ --- __init__.py | 4 +--- config.json | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/__init__.py b/__init__.py index ec6f4b4..7f573e7 100644 --- a/__init__.py +++ b/__init__.py @@ -2,7 +2,6 @@ import asyncio from typing import List from fastapi import APIRouter -from starlette.staticfiles import StaticFiles from lnbits.db import Database from lnbits.helpers import template_renderer @@ -15,7 +14,6 @@ db = Database("ext_nostrclient") nostrclient_static_files = [ { "path": "/nostrclient/static", - "app": StaticFiles(directory="lnbits/extensions/nostrclient/static"), "name": "nostrclient_static", } ] @@ -33,7 +31,7 @@ nostr = NostrClient() def nostr_renderer(): - return template_renderer(["lnbits/extensions/nostrclient/templates"]) + return template_renderer(["nostrclient/templates"]) from .tasks import check_relays, init_relays, subscribe_events diff --git a/config.json b/config.json index ce8ae18..d8b886b 100644 --- a/config.json +++ b/config.json @@ -2,5 +2,6 @@ "name": "Nostr Client", "short_description": "Nostr client for extensions", "tile": "/nostrclient/static/images/nostr-bitcoin.png", - "contributors": ["calle"] + "contributors": ["calle"], + "min_lnbits_version": "0.11.0" } From ab185bd2c426b2115dac7d49dde7118411b78e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Tue, 26 Sep 2023 15:39:33 +0200 Subject: [PATCH 77/96] add release workflow (#22) --- .github/workflows/release.yml | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7ec9b48 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,59 @@ +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + +jobs: + + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Create github release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref_name }} + run: | + gh release create "$tag" --generate-notes + + pullrequest: + needs: [release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.EXT_GITHUB }} + repository: lnbits/lnbits-extensions + path: './lnbits-extensions' + + - name: setup git user + run: | + git config --global user.name "alan" + git config --global user.email "alan@lnbits.com" + + - name: Create pull request in extensions repo + env: + GH_TOKEN: ${{ secrets.EXT_GITHUB }} + repo_name: "${{ github.event.repository.name }}" + tag: "${{ github.ref_name }}" + branch: "update-${{ github.event.repository.name }}-${{ github.ref_name }}" + title: "[UPDATE] ${{ github.event.repository.name }} to ${{ github.ref_name }}" + body: "https://github.com/lnbits/${{ github.event.repository.name }}/releases/${{ github.ref_name }}" + archive: "https://github.com/lnbits/${{ github.event.repository.name }}/archive/refs/tags/${{ github.ref_name }}.zip" + run: | + cd lnbits-extensions + git checkout -b $branch + + # if there is another open PR + git pull origin $branch || echo "branch does not exist" + + sh util.sh update_extension $repo_name $tag + + git add -A + git commit -am "$title" + git push origin $branch + + # check if pr exists before creating it + gh config set pager cat + check=$(gh pr list -H $branch | wc -l) + test $check -ne 0 || gh pr create --title "$title" --body "$body" --repo lnbits/lnbits-extensions From 16ae9d15a143e194deb767c556591307a39f1e5d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 1 Nov 2023 17:46:42 +0200 Subject: [PATCH 78/96] Stabilize (#24) * refactor: clean-up * refactor: extra logs plus try-catch * refactor: do not use bare `except` * refactor: clean-up redundant fields * chore: pass code checks * chore: code format * refactor: code clean-up * fix: refactoring stuff * refactor: remove un-used file * chore: code clean-up * chore: code clean-up * chore: code-format fix * refactor: remove nostr.client wrapper * refactor: code clean-up * chore: code format * refactor: remove `RelayList` class * refactor: extract smaller methods with try-catch * fix: better exception handling * fix: remove redundant filters * fix: simplify event * chore: code format * fix: code check * fix: code check * fix: simplify `REQ` * fix: more clean-ups * refactor: use simpler method * refactor: re-order and rename * fix: stop logic * fix: subscription close before disconnect * chore: play commit --- __init__.py | 11 +- cbc.py | 26 ---- crud.py | 16 +-- migrations.py | 2 +- models.py | 42 ++---- nostr/__init__.py | 0 nostr/bech32.py | 32 +++-- nostr/client/client.py | 57 ++++++-- nostr/delegation.py | 32 ----- nostr/event.py | 3 +- nostr/filter.py | 134 ----------------- nostr/key.py | 14 +- nostr/message_pool.py | 38 ++--- nostr/relay.py | 165 +++++---------------- nostr/relay_manager.py | 109 ++++++++------ nostr/subscription.py | 10 +- router.py | 178 +++++++++-------------- tasks.py | 70 +++++---- templates/nostrclient/index.html | 238 +++++++++++++++++++++++-------- views_api.py | 62 ++++---- 20 files changed, 522 insertions(+), 717 deletions(-) delete mode 100644 cbc.py delete mode 100644 nostr/__init__.py delete mode 100644 nostr/delegation.py delete mode 100644 nostr/filter.py diff --git a/__init__.py b/__init__.py index 7f573e7..d50998b 100644 --- a/__init__.py +++ b/__init__.py @@ -7,7 +7,7 @@ from lnbits.db import Database from lnbits.helpers import template_renderer from lnbits.tasks import catch_everything_and_restart -from .nostr.client.client import NostrClient as NostrClientLib +from .nostr.client.client import NostrClient db = Database("ext_nostrclient") @@ -22,19 +22,14 @@ nostrclient_ext: APIRouter = APIRouter(prefix="/nostrclient", tags=["nostrclient scheduled_tasks: List[asyncio.Task] = [] -class NostrClient: - def __init__(self): - self.client: NostrClientLib = NostrClientLib(connect=False) - - -nostr = NostrClient() +nostr_client = NostrClient() def nostr_renderer(): return template_renderer(["nostrclient/templates"]) -from .tasks import check_relays, init_relays, subscribe_events +from .tasks import check_relays, init_relays, subscribe_events # noqa from .views import * # noqa from .views_api import * # noqa diff --git a/cbc.py b/cbc.py deleted file mode 100644 index 0d9e04f..0000000 --- a/cbc.py +++ /dev/null @@ -1,26 +0,0 @@ -from Cryptodome.Cipher import AES - -BLOCK_SIZE = 16 - - -class AESCipher(object): - """This class is compatible with crypto.createCipheriv('aes-256-cbc')""" - - def __init__(self, key=None): - self.key = key - - def pad(self, data): - length = BLOCK_SIZE - (len(data) % BLOCK_SIZE) - return data + (chr(length) * length).encode() - - def unpad(self, data): - return data[: -(data[-1] if type(data[-1]) == int else ord(data[-1]))] - - def encrypt(self, plain_text): - cipher = AES.new(self.key, AES.MODE_CBC) - b = plain_text.encode("UTF-8") - return cipher.iv, cipher.encrypt(self.pad(b)) - - def decrypt(self, iv, enc_text): - cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) - return self.unpad(cipher.decrypt(enc_text).decode("UTF-8")) diff --git a/crud.py b/crud.py index 780642d..05ca907 100644 --- a/crud.py +++ b/crud.py @@ -1,21 +1,17 @@ -from typing import List, Optional, Union - -import shortuuid - -from lnbits.helpers import urlsafe_short_hash +from typing import List from . import db -from .models import Relay, RelayList +from .models import Relay -async def get_relays() -> RelayList: - row = await db.fetchall("SELECT * FROM nostrclient.relays") - return RelayList(__root__=row) +async def get_relays() -> List[Relay]: + rows = await db.fetchall("SELECT * FROM nostrclient.relays") + return [Relay.from_row(r) for r in rows] async def add_relay(relay: Relay) -> None: await db.execute( - f""" + """ INSERT INTO nostrclient.relays ( id, url, diff --git a/migrations.py b/migrations.py index 5a30e45..73b9ed8 100644 --- a/migrations.py +++ b/migrations.py @@ -3,7 +3,7 @@ async def m001_initial(db): Initial nostrclient table. """ await db.execute( - f""" + """ CREATE TABLE nostrclient.relays ( id TEXT NOT NULL PRIMARY KEY, url TEXT NOT NULL, diff --git a/models.py b/models.py index 88651fc..e08ade3 100644 --- a/models.py +++ b/models.py @@ -1,9 +1,7 @@ -from dataclasses import dataclass -from typing import Dict, List, Optional +from sqlite3 import Row +from typing import List, Optional -from fastapi import Request -from fastapi.param_functions import Query -from pydantic import BaseModel, Field +from pydantic import BaseModel from lnbits.helpers import urlsafe_short_hash @@ -14,7 +12,8 @@ class RelayStatus(BaseModel): error_counter: Optional[int] = 0 error_list: Optional[List] = [] notice_list: Optional[List] = [] - + + class Relay(BaseModel): id: Optional[str] = None url: Optional[str] = None @@ -28,33 +27,9 @@ class Relay(BaseModel): if not self.id: self.id = urlsafe_short_hash() - -class RelayList(BaseModel): - __root__: List[Relay] - - -class Event(BaseModel): - content: str - pubkey: str - created_at: Optional[int] - kind: int - tags: Optional[List[List[str]]] - sig: str - - -class Filter(BaseModel): - ids: Optional[List[str]] - kinds: Optional[List[int]] - authors: Optional[List[str]] - since: Optional[int] - until: Optional[int] - e: Optional[List[str]] = Field(alias="#e") - p: Optional[List[str]] = Field(alias="#p") - limit: Optional[int] - - -class Filters(BaseModel): - __root__: List[Filter] + @classmethod + def from_row(cls, row: Row) -> "Relay": + return cls(**dict(row)) class TestMessage(BaseModel): @@ -62,6 +37,7 @@ class TestMessage(BaseModel): reciever_public_key: str message: str + class TestMessageResponse(BaseModel): private_key: str public_key: str diff --git a/nostr/__init__.py b/nostr/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/nostr/bech32.py b/nostr/bech32.py index 61a92c4..0ae6c80 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -26,19 +26,22 @@ from enum import Enum class Encoding(Enum): """Enumeration type to list the various supported encodings.""" + BECH32 = 1 BECH32M = 2 + CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" -BECH32M_CONST = 0x2bc830a3 +BECH32M_CONST = 0x2BC830A3 + def bech32_polymod(values): """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] + generator = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] chk = 1 for value in values: top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value + chk = (chk & 0x1FFFFFF) << 5 ^ value for i in range(5): chk ^= generator[i] if ((top >> i) & 1) else 0 return chk @@ -58,6 +61,7 @@ def bech32_verify_checksum(hrp, data): return Encoding.BECH32M return None + def bech32_create_checksum(hrp, data, spec): """Compute the checksum values given HRP and data.""" values = bech32_hrp_expand(hrp) + data @@ -69,26 +73,29 @@ def bech32_create_checksum(hrp, data, spec): def bech32_encode(hrp, data, spec): """Compute a Bech32 string given HRP and data values.""" combined = data + bech32_create_checksum(hrp, data, spec) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) + return hrp + "1" + "".join([CHARSET[d] for d in combined]) + def bech32_decode(bech): """Validate a Bech32/Bech32m string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): + if (any(ord(x) < 33 or ord(x) > 126 for x in bech)) or ( + bech.lower() != bech and bech.upper() != bech + ): return (None, None, None) bech = bech.lower() - pos = bech.rfind('1') + pos = bech.rfind("1") if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: return (None, None, None) - if not all(x in CHARSET for x in bech[pos+1:]): + if not all(x in CHARSET for x in bech[pos + 1 :]): return (None, None, None) hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] + data = [CHARSET.find(x) for x in bech[pos + 1 :]] spec = bech32_verify_checksum(hrp, data) if spec is None: return (None, None, None) return (hrp, data[:-6], spec) + def convertbits(data, frombits, tobits, pad=True): """General power-of-2 base conversion.""" acc = 0 @@ -124,7 +131,12 @@ def decode(hrp, addr): return (None, None) if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: return (None, None) - if data[0] == 0 and spec != Encoding.BECH32 or data[0] != 0 and spec != Encoding.BECH32M: + if ( + data[0] == 0 + and spec != Encoding.BECH32 + or data[0] != 0 + and spec != Encoding.BECH32M + ): return (None, None) return (data[0], decoded) diff --git a/nostr/client/client.py b/nostr/client/client.py index db07a06..4624ff3 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,25 +1,36 @@ import asyncio -from typing import List + +from loguru import logger from ..relay_manager import RelayManager class NostrClient: - relays = [ ] relay_manager = RelayManager() - def __init__(self, relays: List[str] = [], connect=True): - if len(relays): - self.relays = relays - if connect: - self.connect() + def __init__(self): + self.running = True - async def connect(self): - for relay in self.relays: - self.relay_manager.add_relay(relay) + def connect(self, relays): + for relay in relays: + try: + self.relay_manager.add_relay(relay) + except Exception as e: + logger.debug(e) + self.running = True + + def reconnect(self, relays): + self.relay_manager.remove_relays() + self.connect(relays) def close(self): - self.relay_manager.close_connections() + try: + self.relay_manager.close_all_subscriptions() + self.relay_manager.close_connections() + + self.running = False + except Exception as e: + logger.error(e) async def subscribe( self, @@ -27,18 +38,36 @@ class NostrClient: callback_notices_func=None, callback_eosenotices_func=None, ): - while True: + while self.running: + self._check_events(callback_events_func) + self._check_notices(callback_notices_func) + self._check_eos_notices(callback_eosenotices_func) + + await asyncio.sleep(0.2) + + def _check_events(self, callback_events_func=None): + try: while self.relay_manager.message_pool.has_events(): event_msg = self.relay_manager.message_pool.get_event() if callback_events_func: callback_events_func(event_msg) + except Exception as e: + logger.debug(e) + + def _check_notices(self, callback_notices_func=None): + try: while self.relay_manager.message_pool.has_notices(): event_msg = self.relay_manager.message_pool.get_notice() if callback_notices_func: callback_notices_func(event_msg) + except Exception as e: + logger.debug(e) + + def _check_eos_notices(self, callback_eosenotices_func=None): + try: while self.relay_manager.message_pool.has_eose_notices(): event_msg = self.relay_manager.message_pool.get_eose_notice() if callback_eosenotices_func: callback_eosenotices_func(event_msg) - - await asyncio.sleep(0.5) + except Exception as e: + logger.debug(e) diff --git a/nostr/delegation.py b/nostr/delegation.py deleted file mode 100644 index 94801f5..0000000 --- a/nostr/delegation.py +++ /dev/null @@ -1,32 +0,0 @@ -import time -from dataclasses import dataclass - - -@dataclass -class Delegation: - delegator_pubkey: str - delegatee_pubkey: str - event_kind: int - duration_secs: int = 30*24*60 # default to 30 days - signature: str = None # set in PrivateKey.sign_delegation - - @property - def expires(self) -> int: - return int(time.time()) + self.duration_secs - - @property - def conditions(self) -> str: - return f"kind={self.event_kind}&created_at<{self.expires}" - - @property - def delegation_token(self) -> str: - return f"nostr:delegation:{self.delegatee_pubkey}:{self.conditions}" - - def get_tag(self) -> list[str]: - """ Called by Event """ - return [ - "delegation", - self.delegator_pubkey, - self.conditions, - self.signature, - ] diff --git a/nostr/event.py b/nostr/event.py index 65b187d..a7d4f1d 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -122,6 +122,7 @@ class EncryptedDirectMessage(Event): def id(self) -> str: if self.content is None: raise Exception( - "EncryptedDirectMessage `id` is undefined until its message is encrypted and stored in the `content` field" + "EncryptedDirectMessage `id` is undefined until its" + + " message is encrypted and stored in the `content` field" ) return super().id diff --git a/nostr/filter.py b/nostr/filter.py deleted file mode 100644 index f119079..0000000 --- a/nostr/filter.py +++ /dev/null @@ -1,134 +0,0 @@ -from collections import UserList -from typing import List - -from .event import Event, EventKind - - -class Filter: - """ - NIP-01 filtering. - - Explicitly supports "#e" and "#p" tag filters via `event_refs` and `pubkey_refs`. - - Arbitrary NIP-12 single-letter tag filters are also supported via `add_arbitrary_tag`. - If a particular single-letter tag gains prominence, explicit support should be - added. For example: - # arbitrary tag - filter.add_arbitrary_tag('t', [hashtags]) - - # promoted to explicit support - Filter(hashtag_refs=[hashtags]) - """ - - def __init__( - self, - event_ids: List[str] = None, - kinds: List[EventKind] = None, - authors: List[str] = None, - since: int = None, - until: int = None, - event_refs: List[ - str - ] = None, # the "#e" attr; list of event ids referenced in an "e" tag - pubkey_refs: List[ - str - ] = None, # The "#p" attr; list of pubkeys referenced in a "p" tag - limit: int = None, - ) -> None: - self.event_ids = event_ids - self.kinds = kinds - self.authors = authors - self.since = since - self.until = until - self.event_refs = event_refs - self.pubkey_refs = pubkey_refs - self.limit = limit - - self.tags = {} - if self.event_refs: - self.add_arbitrary_tag("e", self.event_refs) - if self.pubkey_refs: - self.add_arbitrary_tag("p", self.pubkey_refs) - - def add_arbitrary_tag(self, tag: str, values: list): - """ - Filter on any arbitrary tag with explicit handling for NIP-01 and NIP-12 - single-letter tags. - """ - # NIP-01 'e' and 'p' tags and any NIP-12 single-letter tags must be prefixed with "#" - tag_key = tag if len(tag) > 1 else f"#{tag}" - self.tags[tag_key] = values - - def matches(self, event: Event) -> bool: - if self.event_ids is not None and event.id not in self.event_ids: - return False - if self.kinds is not None and event.kind not in self.kinds: - return False - if self.authors is not None and event.public_key not in self.authors: - return False - if self.since is not None and event.created_at < self.since: - return False - if self.until is not None and event.created_at > self.until: - return False - if (self.event_refs is not None or self.pubkey_refs is not None) and len( - event.tags - ) == 0: - return False - - if self.tags: - e_tag_identifiers = set([e_tag[0] for e_tag in event.tags]) - for f_tag, f_tag_values in self.tags.items(): - # Omit any NIP-01 or NIP-12 "#" chars on single-letter tags - f_tag = f_tag.replace("#", "") - - if f_tag not in e_tag_identifiers: - # Event is missing a tag type that we're looking for - return False - - # Multiple values within f_tag_values are treated as OR search; an Event - # needs to match only one. - # Note: an Event could have multiple entries of the same tag type - # (e.g. a reply to multiple people) so we have to check all of them. - match_found = False - for e_tag in event.tags: - if e_tag[0] == f_tag and e_tag[1] in f_tag_values: - match_found = True - break - if not match_found: - return False - - return True - - def to_json_object(self) -> dict: - res = {} - if self.event_ids is not None: - res["ids"] = self.event_ids - if self.kinds is not None: - res["kinds"] = self.kinds - if self.authors is not None: - res["authors"] = self.authors - if self.since is not None: - res["since"] = self.since - if self.until is not None: - res["until"] = self.until - if self.limit is not None: - res["limit"] = self.limit - if self.tags: - res.update(self.tags) - - return res - - -class Filters(UserList): - def __init__(self, initlist: "list[Filter]" = []) -> None: - super().__init__(initlist) - self.data: "list[Filter]" - - def match(self, event: Event): - for filter in self.data: - if filter.matches(event): - return True - return False - - def to_json_array(self) -> list: - return [filter.to_json_object() for filter in self.data] diff --git a/nostr/key.py b/nostr/key.py index 8089e11..3803650 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,6 +1,5 @@ import base64 import secrets -from hashlib import sha256 import secp256k1 from cffi import FFI @@ -8,7 +7,6 @@ from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from . import bech32 -from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind @@ -37,7 +35,7 @@ class PublicKey: class PrivateKey: def __init__(self, raw_secret: bytes = None) -> None: - if not raw_secret is None: + if raw_secret is not None: self.raw_secret = raw_secret else: self.raw_secret = secrets.token_bytes(32) @@ -79,7 +77,10 @@ class PrivateKey: encryptor = cipher.encryptor() encrypted_message = encryptor.update(padded_data) + encryptor.finalize() - return f"{base64.b64encode(encrypted_message).decode()}?iv={base64.b64encode(iv).decode()}" + return ( + f"{base64.b64encode(encrypted_message).decode()}" + + f"?iv={base64.b64encode(iv).decode()}" + ) def encrypt_dm(self, dm: EncryptedDirectMessage) -> None: dm.content = self.encrypt_message( @@ -116,11 +117,6 @@ class PrivateKey: event.public_key = self.public_key.hex() event.signature = self.sign_message_hash(bytes.fromhex(event.id)) - def sign_delegation(self, delegation: Delegation) -> None: - delegation.signature = self.sign_message_hash( - sha256(delegation.delegation_token.encode()).digest() - ) - def __eq__(self, other): return self.raw_secret == other.raw_secret diff --git a/nostr/message_pool.py b/nostr/message_pool.py index 02f7fd4..a3e6c5f 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -2,13 +2,15 @@ import json from queue import Queue from threading import Lock -from .event import Event from .message_type import RelayMessageType class EventMessage: - def __init__(self, event: Event, subscription_id: str, url: str) -> None: + def __init__( + self, event: str, event_id: str, subscription_id: str, url: str + ) -> None: self.event = event + self.event_id = event_id self.subscription_id = subscription_id self.url = url @@ -59,18 +61,16 @@ class MessagePool: message_type = message_json[0] if message_type == RelayMessageType.EVENT: subscription_id = message_json[1] - e = message_json[2] - event = Event( - e["content"], - e["pubkey"], - e["created_at"], - e["kind"], - e["tags"], - e["sig"], - ) + event = message_json[2] + if "id" not in event: + return + event_id = event["id"] + with self.lock: - if not f"{subscription_id}_{event.id}" in self._unique_events: - self._accept_event(EventMessage(event, subscription_id, url)) + if f"{subscription_id}_{event_id}" not in self._unique_events: + self._accept_event( + EventMessage(json.dumps(event), event_id, subscription_id, url) + ) elif message_type == RelayMessageType.NOTICE: self.notices.put(NoticeMessage(message_json[1], url)) elif message_type == RelayMessageType.END_OF_STORED_EVENTS: @@ -78,10 +78,12 @@ class MessagePool: def _accept_event(self, event_message: EventMessage): """ - Event uniqueness is considered per `subscription_id`. - The `subscription_id` is rewritten to be unique and it is the same accross relays. - The same event can come from different subscriptions (from the same client or from different ones). - Clients that have joined later should receive older events. + Event uniqueness is considered per `subscription_id`. The `subscription_id` is + rewritten to be unique and it is the same accross relays. The same event can + come from different subscriptions (from the same client or from different ones). + Clients that have joined later should receive older events. """ self.events.put(event_message) - self._unique_events.add(f"{event_message.subscription_id}_{event_message.event.id}") \ No newline at end of file + self._unique_events.add( + f"{event_message.subscription_id}_{event_message.event_id}" + ) diff --git a/nostr/relay.py b/nostr/relay.py index caacba0..b576cfa 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,43 +2,23 @@ import asyncio import json import time from queue import Queue -from threading import Lock from typing import List from loguru import logger from websocket import WebSocketApp -from .event import Event -from .filter import Filters from .message_pool import MessagePool -from .message_type import RelayMessageType from .subscription import Subscription -class RelayPolicy: - def __init__(self, should_read: bool = True, should_write: bool = True) -> None: - self.should_read = should_read - self.should_write = should_write - - def to_json_object(self) -> dict[str, bool]: - return {"read": self.should_read, "write": self.should_write} - - class Relay: - def __init__( - self, - url: str, - policy: RelayPolicy, - message_pool: MessagePool, - subscriptions: dict[str, Subscription] = {}, - ) -> None: + def __init__(self, url: str, message_pool: MessagePool) -> None: self.url = url - self.policy = policy self.message_pool = message_pool - self.subscriptions = subscriptions self.connected: bool = False self.reconnect: bool = True self.shutdown: bool = False + self.error_counter: int = 0 self.error_threshold: int = 100 self.error_list: List[str] = [] @@ -47,12 +27,10 @@ class Relay: self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 - self.ssl_options: dict = {} - self.proxy: dict = {} - self.lock = Lock() + self.queue = Queue() - def connect(self, ssl_options: dict = None, proxy: dict = None): + def connect(self): self.ws = WebSocketApp( self.url, on_open=self._on_open, @@ -62,19 +40,14 @@ class Relay: on_ping=self._on_ping, on_pong=self._on_pong, ) - self.ssl_options = ssl_options - self.proxy = proxy if not self.connected: - self.ws.run_forever( - sslopt=ssl_options, - http_proxy_host=None if proxy is None else proxy.get("host"), - http_proxy_port=None if proxy is None else proxy.get("port"), - proxy_type=None if proxy is None else proxy.get("type"), - ping_interval=5, - ) + self.ws.run_forever(ping_interval=10) def close(self): - self.ws.close() + try: + self.ws.close() + except Exception as e: + logger.warning(f"[Relay: {self.url}] Failed to close websocket: {e}") self.connected = False self.shutdown = True @@ -90,10 +63,9 @@ class Relay: def publish(self, message: str): self.queue.put(message) - def publish_subscriptions(self): - for _, subscription in self.subscriptions.items(): - s = subscription.to_json_object() - json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) + def publish_subscriptions(self, subscriptions: List[Subscription] = []): + for s in subscriptions: + json_str = json.dumps(["REQ", s.id] + s.filters) self.publish(json_str) async def queue_worker(self): @@ -103,55 +75,44 @@ class Relay: message = self.queue.get(timeout=1) self.num_sent_events += 1 self.ws.send(message) - except: + except Exception as _: pass else: await asyncio.sleep(1) - - if self.shutdown: - logger.warning(f"Closing queue worker for '{self.url}'.") - break - def add_subscription(self, id, filters: Filters): - with self.lock: - self.subscriptions[id] = Subscription(id, filters) + if self.shutdown: + logger.warning(f"[Relay: {self.url}] Closing queue worker.") + return def close_subscription(self, id: str) -> None: - with self.lock: - self.subscriptions.pop(id) + try: self.publish(json.dumps(["CLOSE", id])) - - def to_json_object(self) -> dict: - return { - "url": self.url, - "policy": self.policy.to_json_object(), - "subscriptions": [ - subscription.to_json_object() - for subscription in self.subscriptions.values() - ], - } + except Exception as e: + logger.debug(f"[Relay: {self.url}] Failed to close subscription: {e}") def add_notice(self, notice: str): - self.notice_list = ([notice] + self.notice_list)[:20] + self.notice_list = [notice] + self.notice_list def _on_open(self, _): - logger.info(f"Connected to relay: '{self.url}'.") + logger.info(f"[Relay: {self.url}] Connected.") self.connected = True - + self.shutdown = False + def _on_close(self, _, status_code, message): - logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") + logger.warning( + f"[Relay: {self.url}] Connection closed." + + f" Status: '{status_code}'. Message: '{message}'." + ) self.close() def _on_message(self, _, message: str): - if self._is_valid_message(message): - self.num_received_events += 1 - self.message_pool.add_message(message, self.url) + self.num_received_events += 1 + self.message_pool.add_message(message, self.url) def _on_error(self, _, error): - logger.warning(f"Relay error: '{str(error)}'") + logger.warning(f"[Relay: {self.url}] Error: '{str(error)}'") self._append_error_message(str(error)) - self.connected = False - self.error_counter += 1 + self.close() def _on_ping(self, *_): return @@ -159,65 +120,7 @@ class Relay: def _on_pong(self, *_): return - def _is_valid_message(self, message: str) -> bool: - message = message.strip("\n") - if not message or message[0] != "[" or message[-1] != "]": - return False - - message_json = json.loads(message) - message_type = message_json[0] - - if not RelayMessageType.is_valid(message_type): - return False - - if message_type == RelayMessageType.EVENT: - return self._is_valid_event_message(message_json) - - if message_type == RelayMessageType.COMMAND_RESULT: - return self._is_valid_command_result_message(message, message_json) - - return True - - def _is_valid_event_message(self, message_json): - if not len(message_json) == 3: - return False - - subscription_id = message_json[1] - with self.lock: - if subscription_id not in self.subscriptions: - return False - - e = message_json[2] - event = Event( - e["content"], - e["pubkey"], - e["created_at"], - e["kind"], - e["tags"], - e["sig"], - ) - if not event.verify(): - return False - - with self.lock: - subscription = self.subscriptions[subscription_id] - - if subscription.filters and not subscription.filters.match(event): - return False - - return True - - def _is_valid_command_result_message(self, message, message_json): - if not len(message_json) < 3: - return False - - if message_json[2] != True: - logger.warning(f"Relay '{self.url}' negative command result: '{message}'") - self._append_error_message(message) - return False - - return True - def _append_error_message(self, message): - self.error_list = ([message] + self.error_list)[:20] - self.last_error_date = int(time.time()) \ No newline at end of file + self.error_counter += 1 + self.error_list = [message] + self.error_list + self.last_error_date = int(time.time()) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index f639fb0..ff7ca9c 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,21 +1,15 @@ - import asyncio -import ssl import threading import time +from typing import List from loguru import logger -from .filter import Filters from .message_pool import MessagePool, NoticeMessage -from .relay import Relay, RelayPolicy +from .relay import Relay from .subscription import Subscription -class RelayException(Exception): - pass - - class RelayManager: def __init__(self) -> None: self.relays: dict[str, Relay] = {} @@ -25,72 +19,97 @@ class RelayManager: self._cached_subscriptions: dict[str, Subscription] = {} self._subscriptions_lock = threading.Lock() - def add_relay(self, url: str, read: bool = True, write: bool = True) -> Relay: + def add_relay(self, url: str) -> Relay: if url in list(self.relays.keys()): - return - - with self._subscriptions_lock: - subscriptions = self._cached_subscriptions.copy() + logger.debug(f"Relay '{url}' already present.") + return self.relays[url] - policy = RelayPolicy(read, write) - relay = Relay(url, policy, self.message_pool, subscriptions) + relay = Relay(url, self.message_pool) self.relays[url] = relay - self._open_connection( - relay, - {"cert_reqs": ssl.CERT_NONE} - ) # NOTE: This disables ssl certificate verification + self._open_connection(relay) - relay.publish_subscriptions() + relay.publish_subscriptions(list(self._cached_subscriptions.values())) return relay def remove_relay(self, url: str): - self.relays[url].close() - self.relays.pop(url) - self.threads[url].join(timeout=5) - self.threads.pop(url) - self.queue_threads[url].join(timeout=5) - self.queue_threads.pop(url) - + try: + self.relays[url].close() + except Exception as e: + logger.debug(e) - def add_subscription(self, id: str, filters: Filters): + if url in self.relays: + self.relays.pop(url) + + try: + self.threads[url].join(timeout=5) + except Exception as e: + logger.debug(e) + + if url in self.threads: + self.threads.pop(url) + + try: + self.queue_threads[url].join(timeout=5) + except Exception as e: + logger.debug(e) + + if url in self.queue_threads: + self.queue_threads.pop(url) + + def remove_relays(self): + relay_urls = list(self.relays.keys()) + for url in relay_urls: + self.remove_relay(url) + + def add_subscription(self, id: str, filters: List[str]): + s = Subscription(id, filters) with self._subscriptions_lock: - self._cached_subscriptions[id] = Subscription(id, filters) + self._cached_subscriptions[id] = s for relay in self.relays.values(): - relay.add_subscription(id, filters) + relay.publish_subscriptions([s]) def close_subscription(self, id: str): - with self._subscriptions_lock: - self._cached_subscriptions.pop(id) + try: + with self._subscriptions_lock: + if id in self._cached_subscriptions: + self._cached_subscriptions.pop(id) - for relay in self.relays.values(): - relay.close_subscription(id) + for relay in self.relays.values(): + relay.close_subscription(id) + except Exception as e: + logger.debug(e) + + def close_subscriptions(self, subscriptions: List[str]): + for id in subscriptions: + self.close_subscription(id) + + def close_all_subscriptions(self): + all_subscriptions = list(self._cached_subscriptions.keys()) + self.close_subscriptions(all_subscriptions) def check_and_restart_relays(self): stopped_relays = [r for r in self.relays.values() if r.shutdown] for relay in stopped_relays: self._restart_relay(relay) - def close_connections(self): for relay in self.relays.values(): relay.close() def publish_message(self, message: str): for relay in self.relays.values(): - if relay.policy.should_write: - relay.publish(message) + relay.publish(message) def handle_notice(self, notice: NoticeMessage): relay = next((r for r in self.relays.values() if r.url == notice.url)) if relay: relay.add_notice(notice.content) - def _open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): + def _open_connection(self, relay: Relay): self.threads[relay.url] = threading.Thread( target=relay.connect, - args=(ssl_options, proxy), name=f"{relay.url}-thread", daemon=True, ) @@ -98,7 +117,7 @@ class RelayManager: def wrap_async_queue_worker(): asyncio.run(relay.queue_worker()) - + self.queue_threads[relay.url] = threading.Thread( target=wrap_async_queue_worker, name=f"{relay.url}-queue", @@ -108,14 +127,16 @@ class RelayManager: def _restart_relay(self, relay: Relay): time_since_last_error = time.time() - relay.last_error_date - - min_wait_time = min(60 * relay.error_counter, 60 * 60 * 24) # try at least once a day + + min_wait_time = min( + 60 * relay.error_counter, 60 * 60 + ) # try at least once an hour if time_since_last_error < min_wait_time: return - + logger.info(f"Restarting connection to relay '{relay.url}'") self.remove_relay(relay.url) new_relay = self.add_relay(relay.url) new_relay.error_counter = relay.error_counter - new_relay.error_list = relay.error_list \ No newline at end of file + new_relay.error_list = relay.error_list diff --git a/nostr/subscription.py b/nostr/subscription.py index 76da0af..a75c1a1 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,13 +1,7 @@ -from .filter import Filters +from typing import List class Subscription: - def __init__(self, id: str, filters: Filters=None) -> None: + def __init__(self, id: str, filters: List[str] = None) -> None: self.id = id self.filters = filters - - def to_json_object(self): - return { - "id": self.id, - "filters": self.filters.to_json_array() - } diff --git a/router.py b/router.py index cc0a380..e6ccdef 100644 --- a/router.py +++ b/router.py @@ -1,42 +1,61 @@ import asyncio import json -from typing import List, Union +from typing import Dict, List -from fastapi import WebSocketDisconnect +from fastapi import WebSocket, WebSocketDisconnect from loguru import logger from lnbits.helpers import urlsafe_short_hash -from . import nostr -from .models import Event, Filter -from .nostr.filter import Filter as NostrFilter -from .nostr.filter import Filters as NostrFilters -from .nostr.message_pool import EndOfStoredEventsMessage, NoticeMessage +from . import nostr_client +from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage class NostrRouter: - - received_subscription_events: dict[str, list[Event]] = {} + received_subscription_events: dict[str, List[EventMessage]] = {} received_subscription_notices: list[NoticeMessage] = [] received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} - def __init__(self, websocket): - self.subscriptions: List[str] = [] + def __init__(self, websocket: WebSocket): self.connected: bool = True - self.websocket = websocket + self.websocket: WebSocket = websocket self.tasks: List[asyncio.Task] = [] - self.original_subscription_ids = {} + self.original_subscription_ids: Dict[str, str] = {} - async def client_to_nostr(self): - """Receives requests / data from the client and forwards it to relays. If the - request was a subscription/filter, registers it with the nostr client lib. - Remembers the subscription id so we can send back responses from the relay to this - client in `nostr_to_client`""" - while True: + @property + def subscriptions(self) -> List[str]: + return list(self.original_subscription_ids.keys()) + + def start(self): + self.connected = True + self.tasks.append(asyncio.create_task(self._client_to_nostr())) + self.tasks.append(asyncio.create_task(self._nostr_to_client())) + + async def stop(self): + nostr_client.relay_manager.close_subscriptions(self.subscriptions) + self.connected = False + + for t in self.tasks: + try: + t.cancel() + except Exception as _: + pass + + try: + await self.websocket.close() + except Exception as _: + pass + + async def _client_to_nostr(self): + """ + Receives requests / data from the client and forwards it to relays. + """ + while self.connected: try: json_str = await self.websocket.receive_text() - except WebSocketDisconnect: - self.connected = False + except WebSocketDisconnect as e: + logger.debug(e) + await self.stop() break try: @@ -44,15 +63,9 @@ class NostrRouter: except Exception as e: logger.debug(f"Failed to handle client message: '{str(e)}'.") - - async def nostr_to_client(self): - """Sends responses from relays back to the client. Polls the subscriptions of this client - stored in `my_subscriptions`. Then gets all responses for this subscription id from `received_subscription_events` which - is filled in tasks.py. Takes one response after the other and relays it back to the client. Reconstructs - the reponse manually because the nostr client lib we're using can't do it. Reconstructs the original subscription id - that we had previously rewritten in order to avoid collisions when multiple clients use the same id. - """ - while True and self.connected: + async def _nostr_to_client(self): + """Sends responses from relays back to the client.""" + while self.connected: try: await self._handle_subscriptions() self._handle_notices() @@ -61,24 +74,6 @@ class NostrRouter: await asyncio.sleep(0.1) - async def start(self): - self.tasks.append(asyncio.create_task(self.client_to_nostr())) - self.tasks.append(asyncio.create_task(self.nostr_to_client())) - - async def stop(self): - for t in self.tasks: - try: - t.cancel() - except: - pass - - for s in self.subscriptions: - try: - nostr.client.relay_manager.close_subscription(s) - except: - pass - self.connected = False - async def _handle_subscriptions(self): for s in self.subscriptions: if s in NostrRouter.received_subscription_events: @@ -86,8 +81,6 @@ class NostrRouter: if s in NostrRouter.received_subscription_eosenotices: await self._handle_received_subscription_eosenotices(s) - - async def _handle_received_subscription_eosenotices(self, s): try: if s not in self.original_subscription_ids: @@ -95,7 +88,7 @@ class NostrRouter: s_original = self.original_subscription_ids[s] event_to_forward = ["EOSE", s_original] del NostrRouter.received_subscription_eosenotices[s] - + await self.websocket.send_text(json.dumps(event_to_forward)) except Exception as e: logger.debug(e) @@ -104,97 +97,62 @@ class NostrRouter: try: if s not in NostrRouter.received_subscription_events: return + while len(NostrRouter.received_subscription_events[s]): - my_event = NostrRouter.received_subscription_events[s].pop(0) - # event.to_message() does not include the subscription ID, we have to add it manually - event_json = { - "id": my_event.id, - "pubkey": my_event.public_key, - "created_at": my_event.created_at, - "kind": my_event.kind, - "tags": my_event.tags, - "content": my_event.content, - "sig": my_event.signature, - } + event_message = NostrRouter.received_subscription_events[s].pop(0) + event_json = event_message.event # this reconstructs the original response from the relay # reconstruct original subscription id s_original = self.original_subscription_ids[s] - event_to_forward = ["EVENT", s_original, event_json] - await self.websocket.send_text(json.dumps(event_to_forward)) + event_to_forward = f"""["EVENT", "{s_original}", {event_json}]""" + await self.websocket.send_text(event_to_forward) except Exception as e: - logger.debug(e) + logger.debug(e) # there are 2900 errors here def _handle_notices(self): while len(NostrRouter.received_subscription_notices): my_event = NostrRouter.received_subscription_notices.pop(0) - # note: we don't send it to the user because we don't know who should receive it - logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") - nostr.client.relay_manager.handle_notice(my_event) - - - - def _marshall_nostr_filters(self, data: Union[dict, list]): - filters = data if isinstance(data, list) else [data] - filters = [Filter.parse_obj(f) for f in filters] - filter_list: list[NostrFilter] = [] - for filter in filters: - filter_list.append( - NostrFilter( - event_ids=filter.ids, # type: ignore - kinds=filter.kinds, # type: ignore - authors=filter.authors, # type: ignore - since=filter.since, # type: ignore - until=filter.until, # type: ignore - event_refs=filter.e, # type: ignore - pubkey_refs=filter.p, # type: ignore - limit=filter.limit, # type: ignore - ) - ) - return NostrFilters(filter_list) + logger.info(f"[Relay '{my_event.url}'] Notice: '{my_event.content}']") + # Note: we don't send it to the user because + # we don't know who should receive it + nostr_client.relay_manager.handle_notice(my_event) async def _handle_client_to_nostr(self, json_str): - """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will - register the subscription in the nostr client library that we're using so we can - receive the callbacks on it later. Will rewrite the subscription id since we expect - multiple clients to use the router and want to avoid subscription id collisions - """ - json_data = json.loads(json_str) - assert len(json_data) - + assert len(json_data), "Bad JSON array" if json_data[0] == "REQ": self._handle_client_req(json_data) return - + if json_data[0] == "CLOSE": self._handle_client_close(json_data[1]) return if json_data[0] == "EVENT": - nostr.client.relay_manager.publish_message(json_str) + nostr_client.relay_manager.publish_message(json_str) return def _handle_client_req(self, json_data): subscription_id = json_data[1] subscription_id_rewritten = urlsafe_short_hash() self.original_subscription_ids[subscription_id_rewritten] = subscription_id - fltr = json_data[2:] - filters = self._marshall_nostr_filters(fltr) + filters = json_data[2:] - nostr.client.relay_manager.add_subscription( - subscription_id_rewritten, filters - ) - request_rewritten = json.dumps([json_data[0], subscription_id_rewritten] + fltr) - - self.subscriptions.append(subscription_id_rewritten) - nostr.client.relay_manager.publish_message(request_rewritten) + nostr_client.relay_manager.add_subscription(subscription_id_rewritten, filters) def _handle_client_close(self, subscription_id): - subscription_id_rewritten = next((k for k, v in self.original_subscription_ids.items() if v == subscription_id), None) + subscription_id_rewritten = next( + ( + k + for k, v in self.original_subscription_ids.items() + if v == subscription_id + ), + None, + ) if subscription_id_rewritten: self.original_subscription_ids.pop(subscription_id_rewritten) - nostr.client.relay_manager.close_subscription(subscription_id_rewritten) + nostr_client.relay_manager.close_subscription(subscription_id_rewritten) else: logger.debug(f"Failed to unsubscribe from '{subscription_id}.'") diff --git a/tasks.py b/tasks.py index 4c316bc..69aa33c 100644 --- a/tasks.py +++ b/tasks.py @@ -3,75 +3,69 @@ import threading from loguru import logger -from . import nostr +from . import nostr_client from .crud import get_relays from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage -from .router import NostrRouter, nostr +from .router import NostrRouter async def init_relays(): - # reinitialize the entire client - nostr.__init__() # get relays from db relays = await get_relays() # set relays and connect to them - nostr.client.relays = list(set([r.url for r in relays.__root__ if r.url])) - await nostr.client.connect() + valid_relays = list(set([r.url for r in relays if r.url])) + + nostr_client.reconnect(valid_relays) async def check_relays(): - """ Check relays that have been disconnected """ + """Check relays that have been disconnected""" while True: try: await asyncio.sleep(20) - nostr.client.relay_manager.check_and_restart_relays() + nostr_client.relay_manager.check_and_restart_relays() except Exception as e: logger.warning(f"Cannot restart relays: '{str(e)}'.") - + async def subscribe_events(): - while not any([r.connected for r in nostr.client.relay_manager.relays.values()]): + while not any([r.connected for r in nostr_client.relay_manager.relays.values()]): await asyncio.sleep(2) def callback_events(eventMessage: EventMessage): - if eventMessage.subscription_id in NostrRouter.received_subscription_events: - # do not add duplicate events (by event id) - if eventMessage.event.id in set( - [ - e.id - for e in NostrRouter.received_subscription_events[eventMessage.subscription_id] - ] - ): - return + sub_id = eventMessage.subscription_id + if sub_id not in NostrRouter.received_subscription_events: + NostrRouter.received_subscription_events[sub_id] = [eventMessage] + return - NostrRouter.received_subscription_events[eventMessage.subscription_id].append( - eventMessage.event - ) - else: - NostrRouter.received_subscription_events[eventMessage.subscription_id] = [ - eventMessage.event - ] - return + # do not add duplicate events (by event id) + ids = set( + [e.event_id for e in NostrRouter.received_subscription_events[sub_id]] + ) + if eventMessage.event_id in ids: + return + + NostrRouter.received_subscription_events[sub_id].append(eventMessage) def callback_notices(noticeMessage: NoticeMessage): if noticeMessage not in NostrRouter.received_subscription_notices: NostrRouter.received_subscription_notices.append(noticeMessage) - return def callback_eose_notices(eventMessage: EndOfStoredEventsMessage): - if eventMessage.subscription_id not in NostrRouter.received_subscription_eosenotices: - NostrRouter.received_subscription_eosenotices[ - eventMessage.subscription_id - ] = eventMessage + sub_id = eventMessage.subscription_id + if sub_id in NostrRouter.received_subscription_eosenotices: + return - return + NostrRouter.received_subscription_eosenotices[sub_id] = eventMessage def wrap_async_subscribe(): - asyncio.run(nostr.client.subscribe( - callback_events, - callback_notices, - callback_eose_notices, - )) + asyncio.run( + nostr_client.subscribe( + callback_events, + callback_notices, + callback_eose_notices, + ) + ) t = threading.Thread( target=wrap_async_subscribe, diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index a0c5999..db0f98e 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -6,13 +6,30 @@
- +
- - - + + @@ -29,18 +46,36 @@
Nostrclient
- +
- +