Merge fort-nix/nix-bitcoin#787: mempool: 2.5.0 -> 3.2.1

7822e2c9d3 mempool: add frontend settings (Erik Arvstedt)
81112a0553 mempool: 2.5.0 -> 3.2.1 (Erik Arvstedt)
9a044fbfed mempool: remove unneded nginx config files (Erik Arvstedt)
710a92d18c mempool: improve comments (Erik Arvstedt)
d61099a535 mempool: minor refactorings (Erik Arvstedt)
c48b99782d mempool: fix version (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 7822e2c9d3

Tree-SHA512: 50f953a324641fa87f913035d99d2a739c33bb8adca20a5337c0061809ff3e17315bb7761655689b791ef9f53fa3995148ca4d4729bf989074cd52ff41b6f03d

dev/dev-features.sh

@@ -293,6 +293,7 @@ c journalctl -u clightning -f
 run-tests.sh -s mempool-regtest container
 
 c systemctl status mempool
+c journalctl -u mempool
 c systemctl status mysql
 c nodeinfo
 
@@ -302,6 +303,8 @@ c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq
 c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq
 
 # Check frontend
+c systemctl status nginx
+c journalctl -u nginx
 c curl -fsS localhost:60845
 c curl -fsS localhost:60845/api/mempool | jq
 c curl -fsS localhost:60845/api/blocks/1 | jq

dev/dev-scenarios.nix

@@ -111,7 +111,10 @@ with lib;
     ];
     services.mempool = {
       enable = true;
-      frontend.address = "0.0.0.0";
+      frontend = {
+        address = "0.0.0.0";
+        settings.LIQUID_ENABLED = true;
+      };
     };
     nix-bitcoin.nodeinfo.enable = true;
   };

modules/mempool.nix

@@ -50,9 +50,23 @@ let
           default = 60845; # A random private port
           description = "HTTP server port.";
         };
+        settings = mkOption {
+          type = with types; attrsOf anything;
+          default = {};
+          example = {
+            TESTNET_ENABLED = true;
+            MEMPOOL_WEBSITE_URL = "mempool.mynode.org";
+          };
+          description = ''
+            Mempool frontend settings.
+            See here for available options:
+            https://github.com/mempool/mempool/blob/master/frontend/src/app/services/state.service.ts
+            (`interface Env` and `defaultEnv`)
+          '';
+        };
         staticContentRoot = mkOption {
           type = types.path;
-          default = nbPkgs.mempool-frontend;
+          default = nbPkgs.mempool-frontend.withConfig cfg.frontend.settings;
           defaultText = "config.nix-bitcoin.pkgs.mempool-frontend";
           description = "
             Path of the static frontend content root.
@@ -106,7 +120,7 @@ let
         };
         description = ''
           Mempool backend settings.
-          See here for possible options:
+          See here for available options:
           https://github.com/mempool/mempool/blob/master/backend/src/config.ts
         '';
       };
@@ -167,10 +181,12 @@ let
     # This must be added to `services.nginx.commonHttpConfig` when
     # `mempool/location-static.conf` is used
     httpConfig = ''
-      include ${nbPkgs.mempool-nginx-conf}/mempool/http-language.conf;
+      include ${nbPkgs.mempool-nginx-conf}/http-language.conf;
     '';
 
-    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`
+    # Config for static website content.
+    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`.
+    # Adapted from mempool/nginx-mempool.conf and mempool/production/nginx/location-redirects.conf
     staticContent = ''
       index index.html;
 
@@ -178,7 +194,7 @@ let
       add_header Vary Accept-Language;
       add_header Vary Cookie;
 
-      include ${nbPkgs.mempool-nginx-conf}/mempool/location-static.conf;
+      include ${nbPkgs.mempool-nginx-conf}/location-static.conf;
 
       # Redirect /api to /docs/api
       location = /api {
@@ -189,7 +205,9 @@ let
       }
     '';
 
-    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`
+    # Config for backend API.
+    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`.
+    # Adapted from mempool/nginx-mempool.conf and mempool/production/nginx/location-api.conf.
     proxyApi = let
       backend = "http://${nbLib.addressWithPort cfg.address cfg.port}";
     in ''
@@ -208,7 +226,7 @@ let
           proxy_set_header Connection "Upgrade";
 
           # Relevant settings from `recommendedProxyConfig` (nixos/nginx/default.nix)
-          # (In the above api locations, this are inherited from the parent scope)
+          # (In the above api locations, these are inherited from the parent scope)
           proxy_set_header Host $host;
           proxy_set_header X-Real-IP $remote_addr;
           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -246,6 +264,7 @@ in {
         HTTP_PORT = cfg.port;
         CACHE_DIR = "${cacheDir}/cache";
         STDOUT_LOG_MIN_PRIORITY = mkDefault "info";
+        AUTOMATIC_POOLS_UPDATE = true;
       };
       CORE_RPC = {
         HOST = bitcoind.rpc.address;
@@ -264,9 +283,10 @@ in {
         ENABLED = true;
         DATABASE = cfg.database.name;
         SOCKET = "/run/mysqld/mysqld.sock";
+        PID_DIR = cacheDir;
       };
     } // optionalAttrs (cfg.tor.proxy) {
-      # Use Tor for rate fetching
+      # Use Tor for rate fetching and pool updating
       SOCKS5PROXY = {
         ENABLED = true;
         USE_ONION = true;

pkgs/default.nix

@@ -21,6 +21,7 @@ let self = {
   inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; })
     mempool-backend
     mempool-frontend
+    mempool-rust-gbt
     mempool-nginx-conf;
   trustedcoin = pkgs.callPackage ./trustedcoin { };
 

pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch

@@ -0,0 +1,42 @@
+From e4b3ebaf0451c1bddbd7dcf8527c296938ebb607 Mon Sep 17 00:00:00 2001
+From: Erik Arvstedt <erik.arvstedt@gmail.com>
+Date: Sun, 1 Jun 2025 11:17:22 +0200
+Subject: [PATCH] allow disabling mining pool fetching in offline environments
+
+Previously, Mempool strictly required fetching mining pool data from
+Github and failed when this was not possible, e.g. in offline
+environments.
+
+This patch allows disabling pool fetching.
+When disabled, empty pool data is inserted into the DB, which
+effectively turns off block pool classification.
+---
+ backend/src/tasks/pools-updater.ts | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts
+index 6b0520dfc..a74259b95 100644
+--- a/backend/src/tasks/pools-updater.ts
++++ b/backend/src/tasks/pools-updater.ts
+@@ -75,7 +75,7 @@ class PoolsUpdater {
+       } else {
+         logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag);
+       }
+-      const poolsJson = await this.query(this.poolsUrl);
++      const poolsJson = (githubSha == "disable-pool-fetching") ? [] : await this.query(this.poolsUrl);
+       if (poolsJson === undefined) {
+         return;
+       }
+@@ -136,6 +136,9 @@ class PoolsUpdater {
+    * Fetch our latest pools-v2.json sha from github
+    */
+   private async fetchPoolsSha(): Promise<string | null> {
++    if (this.poolsUrl == "disable-pool-fetching") {
++      return "disable-pool-fetching";
++    }
+     const response = await this.query(this.treeUrl);
+ 
+     if (response !== undefined) {
+-- 
+2.47.2
+

pkgs/mempool/default.nix

@@ -1,7 +1,7 @@
 { lib
 , stdenvNoCC
-, nodejs-18_x
+, nodejs_22
-, nodejs-slim-18_x
+, nodejs-slim_22
 , fetchFromGitHub
 , fetchNodeModules
 , runCommand
@@ -9,45 +9,55 @@
 , curl
 , cacert
 , rsync
+# for rust-gbt (backend module)
+, cargo
+, rustc
+, rustPlatform
+, napi-rs-cli
 }:
 rec {
-  nodejs = nodejs-18_x;
+  nodejs = nodejs_22;
-  nodejsRuntime = nodejs-slim-18_x;
+  nodejsRuntime = nodejs-slim_22;
+
+  version = "3.2.1";
 
   src = fetchFromGitHub {
     owner = "mempool";
     repo = "mempool";
-    rev = "v2.5.0";
+    tag = "v${version}";
-    hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ=";
+    hash = "sha256-O2XPD1/BXQnzuOP/vMVyRfmFZEgjA85r+PShWne0vqU=";
   };
 
   nodeModules = {
     frontend = fetchNodeModules {
       inherit src nodejs;
-      preBuild = "cd frontend";
+      sourceRoot = "source/frontend";
-      hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI=";
+      hash = "sha256-+jfgsAkDdYvgso8uSHaBj/sQL3fC/ABQWzVTXfdZcU0=";
     };
     backend = fetchNodeModules {
       inherit src nodejs;
-      preBuild = "cd backend";
+      sourceRoot = "source/backend";
-      hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U=";
+      hash = "sha256-y5l2SYZYK9SKSy6g0+mtTWD6JFkkdQHHBboECpEvWZ4=";
     };
   };
 
   frontendAssets = fetchFiles {
     name = "mempool-frontend-assets";
-    hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY=";
+    hash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ=";
     fetcher = ./frontend-assets-fetch.sh;
   };
 
   mempool-backend = mkDerivationMempool {
     pname = "mempool-backend";
 
+    patches = [ ./0001-allow-disabling-mining-pool-fetching.patch ];
+
     buildPhase = ''
       cd backend
       ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules .
       patchShebangs node_modules
 
+      ${sync} ${mempool-rust-gbt}/ rust-gbt
       npm run package
 
       runHook postBuild
@@ -65,10 +75,18 @@ rec {
 
     passthru = {
       inherit nodejs nodejsRuntime;
+      nodeModules = nodeModules.backend;
     };
   };
 
-  mempool-frontend = mkDerivationMempool {
+  mempool-frontend = mkFrontend {};
+
+  # Argument `config` (type: attrset) defines the mempool frontend config.
+  # If `{}`, the default config is used.
+  # See here for available options:
+  # https://github.com/mempool/mempool/blob/master/frontend/src/app/services/state.service.ts
+  # (`interface Env` and `defaultEnv`)
+  mkFrontend = config: mkDerivationMempool {
     pname = "mempool-frontend";
 
     buildPhase = ''
@@ -81,8 +99,10 @@ rec {
       # internet. Disable this script and instead add the assets manually after building.
       : > sync-assets.js
 
-      # If this produces incomplete output (when run in a different build setup),
+      ${lib.optionalString (config != {}) ''
-      # see https://github.com/mempool/mempool/issues/1256
+        ln -s ${builtins.toFile "mempool-frontend-config" (builtins.toJSON config)} mempool-frontend-config.json
+      ''}
+
       npm run build
 
       # Add assets that would otherwise be downloaded by sync-assets.js
@@ -97,19 +117,57 @@ rec {
       runHook postInstall
     '';
 
-    passthru = { assets = frontendAssets; };
+    passthru = {
+      withConfig = mkFrontend;
+      assets = frontendAssets;
+      nodeModules = nodeModules.frontend;
+    };
+  };
+
+  mempool-rust-gbt = stdenvNoCC.mkDerivation rec {
+    pname = "mempool-rust-gbt";
+    inherit version src meta;
+
+    sourceRoot = "source/rust/gbt";
+
+    nativeBuildInputs = [
+      rustPlatform.cargoSetupHook
+      cargo
+      rustc
+      napi-rs-cli
+    ];
+
+    cargoDeps = rustPlatform.fetchCargoVendor {
+      inherit src;
+      name = "${pname}-${version}";
+      inherit sourceRoot;
+      hash = "sha256-eox/K3ipjAqNyFt87lZnxaU/okQLF/KIhqXrX86n+qw=";
+    };
+
+    buildPhase = ''
+      runHook preBuild
+      # napi doesn't accept an absolute path as dest dir, so we can't directly write to $out
+      napi build --platform --release --strip out
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mv out $out
+      cp package.json $out
+    '';
+
+    passthru = { inherit cargoDeps; };
   };
 
   mempool-nginx-conf = runCommand "mempool-nginx-conf" {} ''
     ${sync} --chmod=u+w ${./nginx-conf}/ $out
-    ${sync} ${src}/production/nginx/http-language.conf $out/mempool
+    ${sync} ${src}/production/nginx/http-language.conf $out
   '';
 
   sync = "${rsync}/bin/rsync -a --inplace";
 
   mkDerivationMempool = args: stdenvNoCC.mkDerivation ({
-    version = src.rev;
+    inherit version src meta;
-    inherit src meta;
 
     nativeBuildInputs = [
       makeWrapper

pkgs/mempool/frontend-assets-fetch.sh

@@ -8,8 +8,7 @@ set -euo pipefail
 # This file is updated by ./frontend-assets-update.sh
 
 declare -A revs=(
-    ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5
+    ["mempool/mining-pool-logos"]=53972ebbd08373cf4910cbb3e6421a1f3bba4563
-    ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d
 )
 
 fetchFile() {
@@ -25,7 +24,5 @@ fetchRepo() {
     curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz"
 }
 
-# shellcheck disable=SC2094
-fetchFile "mempool/mining-pools" pools.json > pools.json
 mkdir mining-pools
 fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools

pkgs/mempool/generate.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Use this to start a debug shell at the location of this statement
 # . "${BASH_SOURCE[0]%/*}/../../helper/start-bash-session.sh"
 
-version=2.5.0
+version=3.2.1
 # You can also specify a rev instead:
 # rev=57eddac7f0b99b4fe84d91c0f4a50a4f7ccfe55f
 owner=mempool
@@ -40,9 +40,9 @@ updateSrc() {
     hash=$(nix hash path "$src")
 
     sed -i "
+      s|\bversion = .*;|version = \"$version\";|
       s|\bowner = .*;|owner = \"$owner\";|
-      s|\brev = .*;|rev = \"$rev\";|
+      /fetchFromGitHub/,/hash/ s|\bhash = .*;|hash = \"$hash\";|
-      s|\bhash = .*;|hash = \"$hash\";|
     " default.nix
 }
 
@@ -50,7 +50,7 @@ updateNodeModulesHash() {
     component=$1
     echo
     echo "Fetching node modules for mempool-$component"
-    ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component" "cd $component"
+    ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component".nodeModules "sourceRoot.*$component"
 }
 
 updateFrontendAssets() {
@@ -60,12 +60,19 @@ updateFrontendAssets() {
   ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets"
 }
 
+updateRustGbtCargoDeps() {
+    echo
+    echo "Fetching rust-gbt cargo deps"
+    ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-rust-gbt.cargoDeps "fetchCargoVendor"
+}
+
 if [[ $# == 0 ]]; then
     # Each of these can be run separately
     updateSrc
     updateFrontendAssets
     updateNodeModulesHash backend
     updateNodeModulesHash frontend
+    updateRustGbtCargoDeps
 else
     "$@"
 fi

pkgs/mempool/nginx-conf/location-static.conf

@@ -1,3 +1,6 @@
+# Settings adapted from
+# https://github.com/mempool/mempool/blob/v3.2.1/production/nginx/server-common.conf
+
 # see order of nginx location rules
 # https://stackoverflow.com/questions/5238377/nginx-location-priority
 
@@ -12,7 +15,7 @@ location = / {
 }
 
 # cache /<lang>/main.f40e91d908a068a2.js forever since they never change
-location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) {
+location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ {
 	try_files $uri =404;
 	expires 1y;
 }
@@ -32,7 +35,7 @@ location /resources {
 	expires 1w;
 }
 # cache /main.f40e91d908a068a2.js forever since they never change
-location ~* ^/.+\..+\.(js|css) {
+location ~* ^/.+\..+\.(js|css)$ {
 	try_files /$lang/$uri /en-US/$uri =404;
 	expires 1y;
 }

pkgs/mempool/nginx-conf/mempool/mempool.conf

@@ -1,44 +0,0 @@
-access_log /var/log/nginx/access_mempool.log;
-error_log /var/log/nginx/error_mempool.log;
-
-root /var/www/mempool/browser;
-
-index index.html;
-
-# enable browser and proxy caching
-add_header Cache-Control "public, no-transform";
-
-# vary cache if user changes language preference
-add_header Vary Accept-Language;
-add_header Vary Cookie;
-
-include mempool/location-static.conf;
-
-# static API docs
-location = /api {
-	try_files $uri $uri/ /en-US/index.html =404;
-}
-location = /api/ {
-	try_files $uri $uri/ /en-US/index.html =404;
-}
-
-location /api/v1/ws {
-	proxy_pass http://127.0.0.1:8999/;
-	proxy_http_version 1.1;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "Upgrade";
-}
-location /api/v1 {
-	proxy_pass http://127.0.0.1:8999/api/v1;
-}
-location /api/ {
-	proxy_pass http://127.0.0.1:8999/api/v1/;
-}
-
-# mainnet API
-location /ws {
-	proxy_pass http://127.0.0.1:8999/;
-	proxy_http_version 1.1;
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "Upgrade";
-}

pkgs/mempool/nginx-conf/nginx.conf

@@ -1,82 +0,0 @@
-user nobody;
-pid /var/run/nginx.pid;
-
-worker_processes auto;
-worker_rlimit_nofile 100000;
-
-events {
-    worker_connections 9000;
-    multi_accept on;
-}
-
-http {
-    sendfile on;
-    tcp_nopush on;
-    tcp_nodelay on;
-
-    server_tokens off;
-    server_name_in_redirect off;
-
-    include /etc/nginx/mime.types;
-    default_type application/octet-stream;
-
-    access_log /var/log/nginx/access.log;
-    error_log /var/log/nginx/error.log;
-
-    # reset timed out connections freeing ram
-    reset_timedout_connection on;
-    # maximum time between packets the client can pause when sending nginx any data
-    client_body_timeout 10s;
-    # maximum time the client has to send the entire header to nginx
-    client_header_timeout 10s;
-    # timeout which a single keep-alive client connection will stay open
-    keepalive_timeout 69s;
-    # maximum time between packets nginx is allowed to pause when sending the client data
-    send_timeout 69s;
-
-    # number of requests per connection, does not affect SPDY
-    keepalive_requests 1337;
-
-    # enable gzip compression
-    gzip on;
-    gzip_vary on;
-    gzip_comp_level 6;
-    gzip_min_length 1000;
-    gzip_proxied expired no-cache no-store private auth;
-    # text/html is always compressed by gzip module
-    gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard;
-
-    # limit request body size
-    client_max_body_size 10m;
-
-    # proxy cache
-    proxy_cache off;
-    proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m;
-    types_hash_max_size 2048;
-
-    # exempt localhost from rate limit
-    geo $limited_ip {
-        default        1;
-        127.0.0.1    0;
-    }
-    map $limited_ip $limited_ip_key {
-        1 $binary_remote_addr;
-        0 '';
-    }
-
-    # rate limit requests
-    limit_req_zone $limited_ip_key zone=api:5m rate=200r/m;
-    limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m;
-    limit_req_status 429;
-
-    # rate limit connections
-    limit_conn_zone $limited_ip_key zone=websocket:10m;
-    limit_conn_status 429;
-
-    include mempool/http-language.conf;
-
-    server {
-        listen 127.0.0.1:80;
-        include mempool/mempool.conf;
-    }
-}

test/tests.nix

@@ -85,7 +85,10 @@ let
       '');
 
       tests.mempool = cfg.mempool.enable;
-      services.mempool.electrumServer = "fulcrum";
+      services.mempool = {
+        electrumServer = "fulcrum";
+        settings.MEMPOOL.POOLS_JSON_URL = mkIf config.test.noConnections "disable-pool-fetching";
+      };
 
       tests.lnd = cfg.lnd.enable;
       services.lnd = {