diff --git a/dev/dev-features.sh b/dev/dev-features.sh index 38daff2..3024a94 100644 --- a/dev/dev-features.sh +++ b/dev/dev-features.sh @@ -293,6 +293,7 @@ c journalctl -u clightning -f run-tests.sh -s mempool-regtest container c systemctl status mempool +c journalctl -u mempool c systemctl status mysql c nodeinfo @@ -302,6 +303,8 @@ c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq # Check frontend +c systemctl status nginx +c journalctl -u nginx c curl -fsS localhost:60845 c curl -fsS localhost:60845/api/mempool | jq c curl -fsS localhost:60845/api/blocks/1 | jq diff --git a/dev/dev-scenarios.nix b/dev/dev-scenarios.nix index e6716ca..b633303 100644 --- a/dev/dev-scenarios.nix +++ b/dev/dev-scenarios.nix @@ -111,7 +111,10 @@ with lib; ]; services.mempool = { enable = true; - frontend.address = "0.0.0.0"; + frontend = { + address = "0.0.0.0"; + settings.LIQUID_ENABLED = true; + }; }; nix-bitcoin.nodeinfo.enable = true; }; diff --git a/modules/mempool.nix b/modules/mempool.nix index a74d513..b120380 100644 --- a/modules/mempool.nix +++ b/modules/mempool.nix @@ -50,9 +50,23 @@ let default = 60845; # A random private port description = "HTTP server port."; }; + settings = mkOption { + type = with types; attrsOf anything; + default = {}; + example = { + TESTNET_ENABLED = true; + MEMPOOL_WEBSITE_URL = "mempool.mynode.org"; + }; + description = '' + Mempool frontend settings. + See here for available options: + https://github.com/mempool/mempool/blob/master/frontend/src/app/services/state.service.ts + (`interface Env` and `defaultEnv`) + ''; + }; staticContentRoot = mkOption { type = types.path; - default = nbPkgs.mempool-frontend; + default = nbPkgs.mempool-frontend.withConfig cfg.frontend.settings; defaultText = "config.nix-bitcoin.pkgs.mempool-frontend"; description = " Path of the static frontend content root. @@ -106,7 +120,7 @@ let }; description = '' Mempool backend settings. - See here for possible options: + See here for available options: https://github.com/mempool/mempool/blob/master/backend/src/config.ts ''; }; @@ -167,10 +181,12 @@ let # This must be added to `services.nginx.commonHttpConfig` when # `mempool/location-static.conf` is used httpConfig = '' - include ${nbPkgs.mempool-nginx-conf}/mempool/http-language.conf; + include ${nbPkgs.mempool-nginx-conf}/http-language.conf; ''; - # This should be added to `services.nginx.virtualHosts..extraConfig` + # Config for static website content. + # This should be added to `services.nginx.virtualHosts..extraConfig`. + # Adapted from mempool/nginx-mempool.conf and mempool/production/nginx/location-redirects.conf staticContent = '' index index.html; @@ -178,7 +194,7 @@ let add_header Vary Accept-Language; add_header Vary Cookie; - include ${nbPkgs.mempool-nginx-conf}/mempool/location-static.conf; + include ${nbPkgs.mempool-nginx-conf}/location-static.conf; # Redirect /api to /docs/api location = /api { @@ -189,7 +205,9 @@ let } ''; - # This should be added to `services.nginx.virtualHosts..extraConfig` + # Config for backend API. + # This should be added to `services.nginx.virtualHosts..extraConfig`. + # Adapted from mempool/nginx-mempool.conf and mempool/production/nginx/location-api.conf. proxyApi = let backend = "http://${nbLib.addressWithPort cfg.address cfg.port}"; in '' @@ -208,7 +226,7 @@ let proxy_set_header Connection "Upgrade"; # Relevant settings from `recommendedProxyConfig` (nixos/nginx/default.nix) - # (In the above api locations, this are inherited from the parent scope) + # (In the above api locations, these are inherited from the parent scope) proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -246,6 +264,7 @@ in { HTTP_PORT = cfg.port; CACHE_DIR = "${cacheDir}/cache"; STDOUT_LOG_MIN_PRIORITY = mkDefault "info"; + AUTOMATIC_POOLS_UPDATE = true; }; CORE_RPC = { HOST = bitcoind.rpc.address; @@ -264,9 +283,10 @@ in { ENABLED = true; DATABASE = cfg.database.name; SOCKET = "/run/mysqld/mysqld.sock"; + PID_DIR = cacheDir; }; } // optionalAttrs (cfg.tor.proxy) { - # Use Tor for rate fetching + # Use Tor for rate fetching and pool updating SOCKS5PROXY = { ENABLED = true; USE_ONION = true; diff --git a/pkgs/default.nix b/pkgs/default.nix index 6e6f542..97f705d 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -21,6 +21,7 @@ let self = { inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; }) mempool-backend mempool-frontend + mempool-rust-gbt mempool-nginx-conf; trustedcoin = pkgs.callPackage ./trustedcoin { }; diff --git a/pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch b/pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch new file mode 100644 index 0000000..c9742c8 --- /dev/null +++ b/pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch @@ -0,0 +1,42 @@ +From e4b3ebaf0451c1bddbd7dcf8527c296938ebb607 Mon Sep 17 00:00:00 2001 +From: Erik Arvstedt +Date: Sun, 1 Jun 2025 11:17:22 +0200 +Subject: [PATCH] allow disabling mining pool fetching in offline environments + +Previously, Mempool strictly required fetching mining pool data from +Github and failed when this was not possible, e.g. in offline +environments. + +This patch allows disabling pool fetching. +When disabled, empty pool data is inserted into the DB, which +effectively turns off block pool classification. +--- + backend/src/tasks/pools-updater.ts | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts +index 6b0520dfc..a74259b95 100644 +--- a/backend/src/tasks/pools-updater.ts ++++ b/backend/src/tasks/pools-updater.ts +@@ -75,7 +75,7 @@ class PoolsUpdater { + } else { + logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag); + } +- const poolsJson = await this.query(this.poolsUrl); ++ const poolsJson = (githubSha == "disable-pool-fetching") ? [] : await this.query(this.poolsUrl); + if (poolsJson === undefined) { + return; + } +@@ -136,6 +136,9 @@ class PoolsUpdater { + * Fetch our latest pools-v2.json sha from github + */ + private async fetchPoolsSha(): Promise { ++ if (this.poolsUrl == "disable-pool-fetching") { ++ return "disable-pool-fetching"; ++ } + const response = await this.query(this.treeUrl); + + if (response !== undefined) { +-- +2.47.2 + diff --git a/pkgs/mempool/default.nix b/pkgs/mempool/default.nix index 1a45785..bf7cc83 100644 --- a/pkgs/mempool/default.nix +++ b/pkgs/mempool/default.nix @@ -1,7 +1,7 @@ { lib , stdenvNoCC -, nodejs-18_x -, nodejs-slim-18_x +, nodejs_22 +, nodejs-slim_22 , fetchFromGitHub , fetchNodeModules , runCommand @@ -9,45 +9,55 @@ , curl , cacert , rsync +# for rust-gbt (backend module) +, cargo +, rustc +, rustPlatform +, napi-rs-cli }: rec { - nodejs = nodejs-18_x; - nodejsRuntime = nodejs-slim-18_x; + nodejs = nodejs_22; + nodejsRuntime = nodejs-slim_22; + + version = "3.2.1"; src = fetchFromGitHub { owner = "mempool"; repo = "mempool"; - rev = "v2.5.0"; - hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ="; + tag = "v${version}"; + hash = "sha256-O2XPD1/BXQnzuOP/vMVyRfmFZEgjA85r+PShWne0vqU="; }; nodeModules = { frontend = fetchNodeModules { inherit src nodejs; - preBuild = "cd frontend"; - hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI="; + sourceRoot = "source/frontend"; + hash = "sha256-+jfgsAkDdYvgso8uSHaBj/sQL3fC/ABQWzVTXfdZcU0="; }; backend = fetchNodeModules { inherit src nodejs; - preBuild = "cd backend"; - hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U="; + sourceRoot = "source/backend"; + hash = "sha256-y5l2SYZYK9SKSy6g0+mtTWD6JFkkdQHHBboECpEvWZ4="; }; }; frontendAssets = fetchFiles { name = "mempool-frontend-assets"; - hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY="; + hash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ="; fetcher = ./frontend-assets-fetch.sh; }; mempool-backend = mkDerivationMempool { pname = "mempool-backend"; + patches = [ ./0001-allow-disabling-mining-pool-fetching.patch ]; + buildPhase = '' cd backend ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules . patchShebangs node_modules + ${sync} ${mempool-rust-gbt}/ rust-gbt npm run package runHook postBuild @@ -65,10 +75,18 @@ rec { passthru = { inherit nodejs nodejsRuntime; + nodeModules = nodeModules.backend; }; }; - mempool-frontend = mkDerivationMempool { + mempool-frontend = mkFrontend {}; + + # Argument `config` (type: attrset) defines the mempool frontend config. + # If `{}`, the default config is used. + # See here for available options: + # https://github.com/mempool/mempool/blob/master/frontend/src/app/services/state.service.ts + # (`interface Env` and `defaultEnv`) + mkFrontend = config: mkDerivationMempool { pname = "mempool-frontend"; buildPhase = '' @@ -81,8 +99,10 @@ rec { # internet. Disable this script and instead add the assets manually after building. : > sync-assets.js - # If this produces incomplete output (when run in a different build setup), - # see https://github.com/mempool/mempool/issues/1256 + ${lib.optionalString (config != {}) '' + ln -s ${builtins.toFile "mempool-frontend-config" (builtins.toJSON config)} mempool-frontend-config.json + ''} + npm run build # Add assets that would otherwise be downloaded by sync-assets.js @@ -97,19 +117,57 @@ rec { runHook postInstall ''; - passthru = { assets = frontendAssets; }; + passthru = { + withConfig = mkFrontend; + assets = frontendAssets; + nodeModules = nodeModules.frontend; + }; + }; + + mempool-rust-gbt = stdenvNoCC.mkDerivation rec { + pname = "mempool-rust-gbt"; + inherit version src meta; + + sourceRoot = "source/rust/gbt"; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + cargo + rustc + napi-rs-cli + ]; + + cargoDeps = rustPlatform.fetchCargoVendor { + inherit src; + name = "${pname}-${version}"; + inherit sourceRoot; + hash = "sha256-eox/K3ipjAqNyFt87lZnxaU/okQLF/KIhqXrX86n+qw="; + }; + + buildPhase = '' + runHook preBuild + # napi doesn't accept an absolute path as dest dir, so we can't directly write to $out + napi build --platform --release --strip out + runHook postBuild + ''; + + installPhase = '' + mv out $out + cp package.json $out + ''; + + passthru = { inherit cargoDeps; }; }; mempool-nginx-conf = runCommand "mempool-nginx-conf" {} '' ${sync} --chmod=u+w ${./nginx-conf}/ $out - ${sync} ${src}/production/nginx/http-language.conf $out/mempool + ${sync} ${src}/production/nginx/http-language.conf $out ''; sync = "${rsync}/bin/rsync -a --inplace"; mkDerivationMempool = args: stdenvNoCC.mkDerivation ({ - version = src.rev; - inherit src meta; + inherit version src meta; nativeBuildInputs = [ makeWrapper diff --git a/pkgs/mempool/frontend-assets-fetch.sh b/pkgs/mempool/frontend-assets-fetch.sh index f114e8c..07f69f8 100755 --- a/pkgs/mempool/frontend-assets-fetch.sh +++ b/pkgs/mempool/frontend-assets-fetch.sh @@ -8,8 +8,7 @@ set -euo pipefail # This file is updated by ./frontend-assets-update.sh declare -A revs=( - ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5 - ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d + ["mempool/mining-pool-logos"]=53972ebbd08373cf4910cbb3e6421a1f3bba4563 ) fetchFile() { @@ -25,7 +24,5 @@ fetchRepo() { curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz" } -# shellcheck disable=SC2094 -fetchFile "mempool/mining-pools" pools.json > pools.json mkdir mining-pools fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools diff --git a/pkgs/mempool/generate.sh b/pkgs/mempool/generate.sh index f491a04..5945cab 100755 --- a/pkgs/mempool/generate.sh +++ b/pkgs/mempool/generate.sh @@ -5,7 +5,7 @@ set -euo pipefail # Use this to start a debug shell at the location of this statement # . "${BASH_SOURCE[0]%/*}/../../helper/start-bash-session.sh" -version=2.5.0 +version=3.2.1 # You can also specify a rev instead: # rev=57eddac7f0b99b4fe84d91c0f4a50a4f7ccfe55f owner=mempool @@ -40,9 +40,9 @@ updateSrc() { hash=$(nix hash path "$src") sed -i " + s|\bversion = .*;|version = \"$version\";| s|\bowner = .*;|owner = \"$owner\";| - s|\brev = .*;|rev = \"$rev\";| - s|\bhash = .*;|hash = \"$hash\";| + /fetchFromGitHub/,/hash/ s|\bhash = .*;|hash = \"$hash\";| " default.nix } @@ -50,7 +50,7 @@ updateNodeModulesHash() { component=$1 echo echo "Fetching node modules for mempool-$component" - ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component" "cd $component" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component".nodeModules "sourceRoot.*$component" } updateFrontendAssets() { @@ -60,12 +60,19 @@ updateFrontendAssets() { ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets" } +updateRustGbtCargoDeps() { + echo + echo "Fetching rust-gbt cargo deps" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-rust-gbt.cargoDeps "fetchCargoVendor" +} + if [[ $# == 0 ]]; then # Each of these can be run separately updateSrc updateFrontendAssets updateNodeModulesHash backend updateNodeModulesHash frontend + updateRustGbtCargoDeps else "$@" fi diff --git a/pkgs/mempool/nginx-conf/mempool/location-static.conf b/pkgs/mempool/nginx-conf/location-static.conf similarity index 85% rename from pkgs/mempool/nginx-conf/mempool/location-static.conf rename to pkgs/mempool/nginx-conf/location-static.conf index 2afed65..fd6d4fc 100644 --- a/pkgs/mempool/nginx-conf/mempool/location-static.conf +++ b/pkgs/mempool/nginx-conf/location-static.conf @@ -1,3 +1,6 @@ +# Settings adapted from +# https://github.com/mempool/mempool/blob/v3.2.1/production/nginx/server-common.conf + # see order of nginx location rules # https://stackoverflow.com/questions/5238377/nginx-location-priority @@ -12,7 +15,7 @@ location = / { } # cache //main.f40e91d908a068a2.js forever since they never change -location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { +location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ { try_files $uri =404; expires 1y; } @@ -32,7 +35,7 @@ location /resources { expires 1w; } # cache /main.f40e91d908a068a2.js forever since they never change -location ~* ^/.+\..+\.(js|css) { +location ~* ^/.+\..+\.(js|css)$ { try_files /$lang/$uri /en-US/$uri =404; expires 1y; } diff --git a/pkgs/mempool/nginx-conf/mempool/mempool.conf b/pkgs/mempool/nginx-conf/mempool/mempool.conf deleted file mode 100644 index 78d2813..0000000 --- a/pkgs/mempool/nginx-conf/mempool/mempool.conf +++ /dev/null @@ -1,44 +0,0 @@ -access_log /var/log/nginx/access_mempool.log; -error_log /var/log/nginx/error_mempool.log; - -root /var/www/mempool/browser; - -index index.html; - -# enable browser and proxy caching -add_header Cache-Control "public, no-transform"; - -# vary cache if user changes language preference -add_header Vary Accept-Language; -add_header Vary Cookie; - -include mempool/location-static.conf; - -# static API docs -location = /api { - try_files $uri $uri/ /en-US/index.html =404; -} -location = /api/ { - try_files $uri $uri/ /en-US/index.html =404; -} - -location /api/v1/ws { - proxy_pass http://127.0.0.1:8999/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; -} -location /api/v1 { - proxy_pass http://127.0.0.1:8999/api/v1; -} -location /api/ { - proxy_pass http://127.0.0.1:8999/api/v1/; -} - -# mainnet API -location /ws { - proxy_pass http://127.0.0.1:8999/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; -} diff --git a/pkgs/mempool/nginx-conf/nginx.conf b/pkgs/mempool/nginx-conf/nginx.conf deleted file mode 100644 index cce77d7..0000000 --- a/pkgs/mempool/nginx-conf/nginx.conf +++ /dev/null @@ -1,82 +0,0 @@ -user nobody; -pid /var/run/nginx.pid; - -worker_processes auto; -worker_rlimit_nofile 100000; - -events { - worker_connections 9000; - multi_accept on; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - - server_tokens off; - server_name_in_redirect off; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - # reset timed out connections freeing ram - reset_timedout_connection on; - # maximum time between packets the client can pause when sending nginx any data - client_body_timeout 10s; - # maximum time the client has to send the entire header to nginx - client_header_timeout 10s; - # timeout which a single keep-alive client connection will stay open - keepalive_timeout 69s; - # maximum time between packets nginx is allowed to pause when sending the client data - send_timeout 69s; - - # number of requests per connection, does not affect SPDY - keepalive_requests 1337; - - # enable gzip compression - gzip on; - gzip_vary on; - gzip_comp_level 6; - gzip_min_length 1000; - gzip_proxied expired no-cache no-store private auth; - # text/html is always compressed by gzip module - gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard; - - # limit request body size - client_max_body_size 10m; - - # proxy cache - proxy_cache off; - proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m; - types_hash_max_size 2048; - - # exempt localhost from rate limit - geo $limited_ip { - default 1; - 127.0.0.1 0; - } - map $limited_ip $limited_ip_key { - 1 $binary_remote_addr; - 0 ''; - } - - # rate limit requests - limit_req_zone $limited_ip_key zone=api:5m rate=200r/m; - limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m; - limit_req_status 429; - - # rate limit connections - limit_conn_zone $limited_ip_key zone=websocket:10m; - limit_conn_status 429; - - include mempool/http-language.conf; - - server { - listen 127.0.0.1:80; - include mempool/mempool.conf; - } -} diff --git a/test/tests.nix b/test/tests.nix index 4565059..af6e30c 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -85,7 +85,10 @@ let ''); tests.mempool = cfg.mempool.enable; - services.mempool.electrumServer = "fulcrum"; + services.mempool = { + electrumServer = "fulcrum"; + settings.MEMPOOL.POOLS_JSON_URL = mkIf config.test.noConnections "disable-pool-fetching"; + }; tests.lnd = cfg.lnd.enable; services.lnd = {