From 81112a055323f5c0fb088647999760acfe7fa73b Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 1 Jun 2025 23:03:32 +0200 Subject: [PATCH] mempool: 2.5.0 -> 3.2.1 --- dev/dev-features.sh | 3 + modules/mempool.nix | 4 +- pkgs/default.nix | 1 + ...allow-disabling-mining-pool-fetching.patch | 42 +++++++++++++ pkgs/mempool/default.nix | 63 +++++++++++++++---- pkgs/mempool/frontend-assets-fetch.sh | 5 +- pkgs/mempool/generate.sh | 9 ++- pkgs/mempool/nginx-conf/location-static.conf | 7 ++- test/tests.nix | 5 +- 9 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch diff --git a/dev/dev-features.sh b/dev/dev-features.sh index 38daff2..3024a94 100644 --- a/dev/dev-features.sh +++ b/dev/dev-features.sh @@ -293,6 +293,7 @@ c journalctl -u clightning -f run-tests.sh -s mempool-regtest container c systemctl status mempool +c journalctl -u mempool c systemctl status mysql c nodeinfo @@ -302,6 +303,8 @@ c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq # Check frontend +c systemctl status nginx +c journalctl -u nginx c curl -fsS localhost:60845 c curl -fsS localhost:60845/api/mempool | jq c curl -fsS localhost:60845/api/blocks/1 | jq diff --git a/modules/mempool.nix b/modules/mempool.nix index a14be03..86648ad 100644 --- a/modules/mempool.nix +++ b/modules/mempool.nix @@ -250,6 +250,7 @@ in { HTTP_PORT = cfg.port; CACHE_DIR = "${cacheDir}/cache"; STDOUT_LOG_MIN_PRIORITY = mkDefault "info"; + AUTOMATIC_POOLS_UPDATE = true; }; CORE_RPC = { HOST = bitcoind.rpc.address; @@ -268,9 +269,10 @@ in { ENABLED = true; DATABASE = cfg.database.name; SOCKET = "/run/mysqld/mysqld.sock"; + PID_DIR = cacheDir; }; } // optionalAttrs (cfg.tor.proxy) { - # Use Tor for rate fetching + # Use Tor for rate fetching and pool updating SOCKS5PROXY = { ENABLED = true; USE_ONION = true; diff --git a/pkgs/default.nix b/pkgs/default.nix index 6e6f542..97f705d 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -21,6 +21,7 @@ let self = { inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; }) mempool-backend mempool-frontend + mempool-rust-gbt mempool-nginx-conf; trustedcoin = pkgs.callPackage ./trustedcoin { }; diff --git a/pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch b/pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch new file mode 100644 index 0000000..c9742c8 --- /dev/null +++ b/pkgs/mempool/0001-allow-disabling-mining-pool-fetching.patch @@ -0,0 +1,42 @@ +From e4b3ebaf0451c1bddbd7dcf8527c296938ebb607 Mon Sep 17 00:00:00 2001 +From: Erik Arvstedt +Date: Sun, 1 Jun 2025 11:17:22 +0200 +Subject: [PATCH] allow disabling mining pool fetching in offline environments + +Previously, Mempool strictly required fetching mining pool data from +Github and failed when this was not possible, e.g. in offline +environments. + +This patch allows disabling pool fetching. +When disabled, empty pool data is inserted into the DB, which +effectively turns off block pool classification. +--- + backend/src/tasks/pools-updater.ts | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts +index 6b0520dfc..a74259b95 100644 +--- a/backend/src/tasks/pools-updater.ts ++++ b/backend/src/tasks/pools-updater.ts +@@ -75,7 +75,7 @@ class PoolsUpdater { + } else { + logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag); + } +- const poolsJson = await this.query(this.poolsUrl); ++ const poolsJson = (githubSha == "disable-pool-fetching") ? [] : await this.query(this.poolsUrl); + if (poolsJson === undefined) { + return; + } +@@ -136,6 +136,9 @@ class PoolsUpdater { + * Fetch our latest pools-v2.json sha from github + */ + private async fetchPoolsSha(): Promise { ++ if (this.poolsUrl == "disable-pool-fetching") { ++ return "disable-pool-fetching"; ++ } + const response = await this.query(this.treeUrl); + + if (response !== undefined) { +-- +2.47.2 + diff --git a/pkgs/mempool/default.nix b/pkgs/mempool/default.nix index 727826f..9ba4aba 100644 --- a/pkgs/mempool/default.nix +++ b/pkgs/mempool/default.nix @@ -1,7 +1,7 @@ { lib , stdenvNoCC -, nodejs-18_x -, nodejs-slim-18_x +, nodejs_22 +, nodejs-slim_22 , fetchFromGitHub , fetchNodeModules , runCommand @@ -9,47 +9,55 @@ , curl , cacert , rsync +# for rust-gbt (backend module) +, cargo +, rustc +, rustPlatform +, napi-rs-cli }: rec { - nodejs = nodejs-18_x; - nodejsRuntime = nodejs-slim-18_x; + nodejs = nodejs_22; + nodejsRuntime = nodejs-slim_22; - version = "2.5.0"; + version = "3.2.1"; src = fetchFromGitHub { owner = "mempool"; repo = "mempool"; tag = "v${version}"; - hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ="; + hash = "sha256-O2XPD1/BXQnzuOP/vMVyRfmFZEgjA85r+PShWne0vqU="; }; nodeModules = { frontend = fetchNodeModules { inherit src nodejs; sourceRoot = "source/frontend"; - hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI="; + hash = "sha256-+jfgsAkDdYvgso8uSHaBj/sQL3fC/ABQWzVTXfdZcU0="; }; backend = fetchNodeModules { inherit src nodejs; sourceRoot = "source/backend"; - hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U="; + hash = "sha256-y5l2SYZYK9SKSy6g0+mtTWD6JFkkdQHHBboECpEvWZ4="; }; }; frontendAssets = fetchFiles { name = "mempool-frontend-assets"; - hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY="; + hash = "sha256-r6GfOY8Pdh15o2OQMk8syfvWMV6WMCReToAEkQm7tqQ="; fetcher = ./frontend-assets-fetch.sh; }; mempool-backend = mkDerivationMempool { pname = "mempool-backend"; + patches = [ ./0001-allow-disabling-mining-pool-fetching.patch ]; + buildPhase = '' cd backend ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules . patchShebangs node_modules + ${sync} ${mempool-rust-gbt}/ rust-gbt npm run package runHook postBuild @@ -84,8 +92,6 @@ rec { # internet. Disable this script and instead add the assets manually after building. : > sync-assets.js - # If this produces incomplete output (when run in a different build setup), - # see https://github.com/mempool/mempool/issues/1256 npm run build # Add assets that would otherwise be downloaded by sync-assets.js @@ -106,6 +112,41 @@ rec { }; }; + mempool-rust-gbt = stdenvNoCC.mkDerivation rec { + pname = "mempool-rust-gbt"; + inherit version src meta; + + sourceRoot = "source/rust/gbt"; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + cargo + rustc + napi-rs-cli + ]; + + cargoDeps = rustPlatform.fetchCargoVendor { + inherit src; + name = "${pname}-${version}"; + inherit sourceRoot; + hash = "sha256-eox/K3ipjAqNyFt87lZnxaU/okQLF/KIhqXrX86n+qw="; + }; + + buildPhase = '' + runHook preBuild + # napi doesn't accept an absolute path as dest dir, so we can't directly write to $out + napi build --platform --release --strip out + runHook postBuild + ''; + + installPhase = '' + mv out $out + cp package.json $out + ''; + + passthru = { inherit cargoDeps; }; + }; + mempool-nginx-conf = runCommand "mempool-nginx-conf" {} '' ${sync} --chmod=u+w ${./nginx-conf}/ $out ${sync} ${src}/production/nginx/http-language.conf $out diff --git a/pkgs/mempool/frontend-assets-fetch.sh b/pkgs/mempool/frontend-assets-fetch.sh index f114e8c..07f69f8 100755 --- a/pkgs/mempool/frontend-assets-fetch.sh +++ b/pkgs/mempool/frontend-assets-fetch.sh @@ -8,8 +8,7 @@ set -euo pipefail # This file is updated by ./frontend-assets-update.sh declare -A revs=( - ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5 - ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d + ["mempool/mining-pool-logos"]=53972ebbd08373cf4910cbb3e6421a1f3bba4563 ) fetchFile() { @@ -25,7 +24,5 @@ fetchRepo() { curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz" } -# shellcheck disable=SC2094 -fetchFile "mempool/mining-pools" pools.json > pools.json mkdir mining-pools fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools diff --git a/pkgs/mempool/generate.sh b/pkgs/mempool/generate.sh index a70a592..5945cab 100755 --- a/pkgs/mempool/generate.sh +++ b/pkgs/mempool/generate.sh @@ -5,7 +5,7 @@ set -euo pipefail # Use this to start a debug shell at the location of this statement # . "${BASH_SOURCE[0]%/*}/../../helper/start-bash-session.sh" -version=2.5.0 +version=3.2.1 # You can also specify a rev instead: # rev=57eddac7f0b99b4fe84d91c0f4a50a4f7ccfe55f owner=mempool @@ -60,12 +60,19 @@ updateFrontendAssets() { ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets" } +updateRustGbtCargoDeps() { + echo + echo "Fetching rust-gbt cargo deps" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-rust-gbt.cargoDeps "fetchCargoVendor" +} + if [[ $# == 0 ]]; then # Each of these can be run separately updateSrc updateFrontendAssets updateNodeModulesHash backend updateNodeModulesHash frontend + updateRustGbtCargoDeps else "$@" fi diff --git a/pkgs/mempool/nginx-conf/location-static.conf b/pkgs/mempool/nginx-conf/location-static.conf index 2afed65..fd6d4fc 100644 --- a/pkgs/mempool/nginx-conf/location-static.conf +++ b/pkgs/mempool/nginx-conf/location-static.conf @@ -1,3 +1,6 @@ +# Settings adapted from +# https://github.com/mempool/mempool/blob/v3.2.1/production/nginx/server-common.conf + # see order of nginx location rules # https://stackoverflow.com/questions/5238377/nginx-location-priority @@ -12,7 +15,7 @@ location = / { } # cache //main.f40e91d908a068a2.js forever since they never change -location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { +location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ { try_files $uri =404; expires 1y; } @@ -32,7 +35,7 @@ location /resources { expires 1w; } # cache /main.f40e91d908a068a2.js forever since they never change -location ~* ^/.+\..+\.(js|css) { +location ~* ^/.+\..+\.(js|css)$ { try_files /$lang/$uri /en-US/$uri =404; expires 1y; } diff --git a/test/tests.nix b/test/tests.nix index 4565059..af6e30c 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -85,7 +85,10 @@ let ''); tests.mempool = cfg.mempool.enable; - services.mempool.electrumServer = "fulcrum"; + services.mempool = { + electrumServer = "fulcrum"; + settings.MEMPOOL.POOLS_JSON_URL = mkIf config.test.noConnections "disable-pool-fetching"; + }; tests.lnd = cfg.lnd.enable; services.lnd = {