mempool: 2.5.0 -> 3.2.1

dev/dev-features.sh

@@ -293,6 +293,7 @@ c journalctl -u clightning -f
 run-tests.sh -s mempool-regtest container
 
 c systemctl status mempool
+c journalctl -u mempool
 c systemctl status mysql
 c nodeinfo
 
@@ -302,6 +303,8 @@ c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq
 c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq
 
 # Check frontend
+c systemctl status nginx
+c journalctl -u nginx
 c curl -fsS localhost:60845
 c curl -fsS localhost:60845/api/mempool | jq
 c curl -fsS localhost:60845/api/blocks/1 | jq

modules/mempool.nix

@@ -250,6 +250,7 @@ in {
         HTTP_PORT = cfg.port;
         CACHE_DIR = "${cacheDir}/cache";
         STDOUT_LOG_MIN_PRIORITY = mkDefault "info";
+        AUTOMATIC_POOLS_UPDATE = true;
       };
       CORE_RPC = {
         HOST = bitcoind.rpc.address;
@@ -268,9 +269,10 @@ in {
         ENABLED = true;
         DATABASE = cfg.database.name;
         SOCKET = "/run/mysqld/mysqld.sock";
+        PID_DIR = cacheDir;
       };
     } // optionalAttrs (cfg.tor.proxy) {
-      # Use Tor for rate fetching
+      # Use Tor for rate fetching and pool updating
       SOCKS5PROXY = {
         ENABLED = true;
         USE_ONION = true;

pkgs/default.nix

@@ -21,6 +21,7 @@ let self = {
   inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; })
     mempool-backend
     mempool-frontend
+    mempool-rust-gbt
     mempool-nginx-conf;
   trustedcoin = pkgs.callPackage ./trustedcoin { };
 

pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch

@@ -0,0 +1,42 @@
+From e4b3ebaf0451c1bddbd7dcf8527c296938ebb607 Mon Sep 17 00:00:00 2001
+From: Erik Arvstedt <erik.arvstedt@gmail.com>
+Date: Sun, 1 Jun 2025 11:17:22 +0200
+Subject: [PATCH] allow disabling mining pool fetching in offline environments
+
+Previously, Mempool strictly required fetching mining pool data from
+Github and failed when this was not possible, e.g. in offline
+environments.
+
+This patch allows disabling pool fetching.
+When disabled, empty pool data is inserted into the DB, which
+effectively turns off block pool classification.
+---
+ backend/src/tasks/pools-updater.ts | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts
+index 6b0520dfc..a74259b95 100644
+--- a/backend/src/tasks/pools-updater.ts
++++ b/backend/src/tasks/pools-updater.ts
+@@ -75,7 +75,7 @@ class PoolsUpdater {
+       } else {
+         logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag);
+       }
+-      const poolsJson = await this.query(this.poolsUrl);
++      const poolsJson = (githubSha == "disable-pool-fetching") ? [] : await this.query(this.poolsUrl);
+       if (poolsJson === undefined) {
+         return;
+       }
+@@ -136,6 +136,9 @@ class PoolsUpdater {
+    * Fetch our latest pools-v2.json sha from github
+    */
+   private async fetchPoolsSha(): Promise<string | null> {
++    if (this.poolsUrl == "disable-pool-fetching") {
++      return "disable-pool-fetching";
++    }
+     const response = await this.query(this.treeUrl);
+ 
+     if (response !== undefined) {
+-- 
+2.47.2
+

pkgs/mempool/default.nix

@@ -1,7 +1,7 @@
 { lib
 , stdenvNoCC
-, nodejs-18_x
-, nodejs-slim-18_x
+, nodejs_22
+, nodejs-slim_22
 , fetchFromGitHub
 , fetchNodeModules
 , runCommand
@@ -9,47 +9,55 @@
 , curl
 , cacert
 , rsync
+# for rust-gbt (backend module)
+, cargo
+, rustc
+, rustPlatform
+, napi-rs-cli
 }:
 rec {
-  nodejs = nodejs-18_x;
-  nodejsRuntime = nodejs-slim-18_x;
+  nodejs = nodejs_22;
+  nodejsRuntime = nodejs-slim_22;
 
-  version = "2.5.0";
+  version = "3.2.1";
 
   src = fetchFromGitHub {
     owner = "mempool";
     repo = "mempool";
     tag = "v${version}";
-    hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ=";
+    hash = "sha256-O2XPD1/BXQnzuOP/vMVyRfmFZEgjA85r+PShWne0vqU=";
   };
 
   nodeModules = {
     frontend = fetchNodeModules {
       inherit src nodejs;
       sourceRoot = "source/frontend";
-      hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI=";
+      hash = "sha256-+jfgsAkDdYvgso8uSHaBj/sQL3fC/ABQWzVTXfdZcU0=";
     };
     backend = fetchNodeModules {
       inherit src nodejs;
       sourceRoot = "source/backend";
-      hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U=";
+      hash = "sha256-y5l2SYZYK9SKSy6g0+mtTWD6JFkkdQHHBboECpEvWZ4=";
     };
   };
 
   frontendAssets = fetchFiles {
     name = "mempool-frontend-assets";
-    hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY=";
+    hash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ=";
     fetcher = ./frontend-assets-fetch.sh;
   };
 
   mempool-backend = mkDerivationMempool {
     pname = "mempool-backend";
 
+    patches = [ ./0001-allow-disabling-mining-pool-fetching.patch ];
+
     buildPhase = ''
       cd backend
       ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules .
       patchShebangs node_modules
 
+      ${sync} ${mempool-rust-gbt}/ rust-gbt
       npm run package
 
       runHook postBuild
@@ -84,8 +92,6 @@ rec {
       # internet. Disable this script and instead add the assets manually after building.
       : > sync-assets.js
 
-      # If this produces incomplete output (when run in a different build setup),
-      # see https://github.com/mempool/mempool/issues/1256
       npm run build
 
       # Add assets that would otherwise be downloaded by sync-assets.js
@@ -106,6 +112,41 @@ rec {
     };
   };
 
+  mempool-rust-gbt = stdenvNoCC.mkDerivation rec {
+    pname = "mempool-rust-gbt";
+    inherit version src meta;
+
+    sourceRoot = "source/rust/gbt";
+
+    nativeBuildInputs = [
+      rustPlatform.cargoSetupHook
+      cargo
+      rustc
+      napi-rs-cli
+    ];
+
+    cargoDeps = rustPlatform.fetchCargoVendor {
+      inherit src;
+      name = "${pname}-${version}";
+      inherit sourceRoot;
+      hash = "sha256-eox/K3ipjAqNyFt87lZnxaU/okQLF/KIhqXrX86n+qw=";
+    };
+
+    buildPhase = ''
+      runHook preBuild
+      # napi doesn't accept an absolute path as dest dir, so we can't directly write to $out
+      napi build --platform --release --strip out
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mv out $out
+      cp package.json $out
+    '';
+
+    passthru = { inherit cargoDeps; };
+  };
+
   mempool-nginx-conf = runCommand "mempool-nginx-conf" {} ''
     ${sync} --chmod=u+w ${./nginx-conf}/ $out
     ${sync} ${src}/production/nginx/http-language.conf $out

pkgs/mempool/frontend-assets-fetch.sh

@@ -8,8 +8,7 @@ set -euo pipefail
 # This file is updated by ./frontend-assets-update.sh
 
 declare -A revs=(
-    ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5
-    ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d
+    ["mempool/mining-pool-logos"]=53972ebbd08373cf4910cbb3e6421a1f3bba4563
 )
 
 fetchFile() {
@@ -25,7 +24,5 @@ fetchRepo() {
     curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz"
 }
 
-# shellcheck disable=SC2094
-fetchFile "mempool/mining-pools" pools.json > pools.json
 mkdir mining-pools
 fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools

pkgs/mempool/generate.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Use this to start a debug shell at the location of this statement
 # . "${BASH_SOURCE[0]%/*}/../../helper/start-bash-session.sh"
 
-version=2.5.0
+version=3.2.1
 # You can also specify a rev instead:
 # rev=57eddac7f0b99b4fe84d91c0f4a50a4f7ccfe55f
 owner=mempool
@@ -60,12 +60,19 @@ updateFrontendAssets() {
   ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets"
 }
 
+updateRustGbtCargoDeps() {
+    echo
+    echo "Fetching rust-gbt cargo deps"
+    ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-rust-gbt.cargoDeps "fetchCargoVendor"
+}
+
 if [[ $# == 0 ]]; then
     # Each of these can be run separately
     updateSrc
     updateFrontendAssets
     updateNodeModulesHash backend
     updateNodeModulesHash frontend
+    updateRustGbtCargoDeps
 else
     "$@"
 fi

pkgs/mempool/nginx-conf/location-static.conf

@@ -1,3 +1,6 @@
+# Settings adapted from
+# https://github.com/mempool/mempool/blob/v3.2.1/production/nginx/server-common.conf
+
 # see order of nginx location rules
 # https://stackoverflow.com/questions/5238377/nginx-location-priority
 
@@ -12,7 +15,7 @@ location = / {
 }
 
 # cache /<lang>/main.f40e91d908a068a2.js forever since they never change
-location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) {
+location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ {
 	try_files $uri =404;
 	expires 1y;
 }
@@ -32,7 +35,7 @@ location /resources {
 	expires 1w;
 }
 # cache /main.f40e91d908a068a2.js forever since they never change
-location ~* ^/.+\..+\.(js|css) {
+location ~* ^/.+\..+\.(js|css)$ {
 	try_files /$lang/$uri /en-US/$uri =404;
 	expires 1y;
 }

test/tests.nix

@@ -85,7 +85,10 @@ let
       '');
 
       tests.mempool = cfg.mempool.enable;
-      services.mempool.electrumServer = "fulcrum";
+      services.mempool = {
+        electrumServer = "fulcrum";
+        settings.MEMPOOL.POOLS_JSON_URL = mkIf config.test.noConnections "disable-pool-fetching";
+      };
 
       tests.lnd = cfg.lnd.enable;
       services.lnd = {