fix python packages for nixos 23.05

pkgs/clightning-plugins/default.nix

@@ -42,7 +42,7 @@ let
       description = "Lightning node exporter for the prometheus timeseries server";
       extraPkgs = [ prometheus_client ];
       patchRequirements =
-        "--replace prometheus-client==0.6.0 prometheus-client==0.15.0"
+        "--replace prometheus-client==0.6.0 prometheus-client==0.16.0"
         + " --replace pyln-client~=0.9.3 pyln-client~=23.02";
     };
     rebalance = {
@@ -79,7 +79,7 @@ let
 
         # Check that requirements are met
         PYTHONPATH='${toString python}/${python.sitePackages}' \
-          ${pkgs.python3Packages.pip}/bin/pip install -r requirements.txt --no-cache --no-index
+          ${pkgs.python3Packages.pip}/bin/pip install -r requirements.txt --no-cache --no-index --break-system-packages
 
         chmod +x '${script}'
         patchShebangs '${script}'

pkgs/python-packages/default.nix

@@ -1,4 +1,15 @@
 nbPkgs: python3:
+let
+  # Ignore eval error:
+  # `OpenSSL 1.1 is reaching its end of life on 2023/09/11 and cannot
+  # be supported through the NixOS 23.05 release cycle.`
+  # TODO-EXTERNAL: consider removing when
+  # https://github.com/Simplexum/python-bitcointx/issues/76 and
+  # https://github.com/JoinMarket-Org/joinmarket-clientserver#1451 are resolved.
+  openssl_1_1 = python3.pkgs.pkgs.openssl_1_1.overrideAttrs (old: {
+    meta = builtins.removeAttrs old.meta [ "knownVulnerabilities" ];
+  });
+in
 rec {
   pyPkgsOverrides = self: super: let
     inherit (self) callPackage;
@@ -13,12 +24,15 @@ rec {
       pyln-bolt7 = clightningPkg ./pyln-bolt7;
       pylightning = clightningPkg ./pylightning;
 
+      # bitstring 3.1.9, required by pyln-proto
+      bitstring = callPackage ./specific-versions/bitstring.nix {};
+
       # Packages only used by joinmarket
       bencoderpyx = callPackage ./bencoderpyx {};
       chromalog = callPackage ./chromalog {};
       python-bitcointx = callPackage ./python-bitcointx {
         inherit (nbPkgs) secp256k1;
-        openssl = super.pkgs.openssl_1_1;
+        openssl = openssl_1_1;
       };
       runes = callPackage ./runes {};
       sha256 = callPackage ./sha256 {};
@@ -39,7 +53,7 @@ rec {
 
       # cryptography 3.3.2, required by joinmarketdaemon
       cryptography = callPackage ./specific-versions/cryptography {
-        openssl = super.pkgs.openssl_1_1;
+        openssl = openssl_1_1;
         cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {};
       };
 
@@ -49,8 +63,8 @@ rec {
       # pyopenssl 21.0.0, required by joinmarketdaemon
       pyopenssl = callPackage ./specific-versions/pyopenssl.nix {};
 
-      # twisted 22.4.0, required by joinmarketbase
+      # txtorcon 22.0.0, required by joinmarketdaemon
-      twisted = callPackage ./specific-versions/twisted.nix {};
+      txtorcon = callPackage ./specific-versions/txtorcon.nix {};
     };
 
   nbPython3Packages = (python3.override {

pkgs/python-packages/jmbase/default.nix

@@ -6,6 +6,10 @@ buildPythonPackage rec {
 
   postUnpack = "sourceRoot=$sourceRoot/jmbase";
 
+  patchPhase = ''
+    sed -i 's|twisted==22.4.0|twisted==22.10.0|' setup.py
+  '';
+
   propagatedBuildInputs = [ future twisted service-identity chromalog txtorcon ];
 
   meta = with lib; {

pkgs/python-packages/jmclient/default.nix

@@ -14,7 +14,7 @@ buildPythonPackage rec {
     substituteInPlace setup.py \
       --replace "'klein==20.6.0'" "'klein>=20.6.0'"
     substituteInPlace setup.py \
-      --replace "'pyjwt==2.4.0'" "'pyjwt==2.5.0'"
+      --replace "'pyjwt==2.4.0'" "'pyjwt==2.6.0'"
   '';
 
   meta = with lib; {

pkgs/python-packages/pyln-proto/default.nix

@@ -29,6 +29,6 @@ buildPythonPackage rec {
   postUnpack = "sourceRoot=$sourceRoot/contrib/pyln-proto";
 
   postPatch = ''
-    sed -i 's|cryptography = "^36.0.1"|cryptography = "^38.0.0"|' pyproject.toml
+    sed -i 's|cryptography = "^36.0.1"|cryptography = "^40.0"|' pyproject.toml
   '';
 }

pkgs/python-packages/specific-versions/bitstring.nix

@@ -0,0 +1,40 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, fetchpatch
+, unittestCheckHook
+}:
+
+buildPythonPackage rec {
+  pname = "bitstring";
+  version = "3.1.9";
+
+  src = fetchFromGitHub {
+    owner = "scott-griffiths";
+    repo = pname;
+    rev = "bitstring-${version}";
+    sha256 = "0y2kcq58psvl038r6dhahhlhp1wjgr5zsms45wyz1naq6ri8x9qa";
+  };
+
+  patches = [
+    (fetchpatch {
+      name = "fix-running-unit-tests-using-unittest-hook.patch";
+      url = "https://github.com/scott-griffiths/bitstring/commit/e5ee3fd41cad2ea761f4450b13b0424ae7262331.patch";
+      hash = "sha256-+ZGywIfQQcYXJlYZBi402ONnysYm66G5zE4duJE40h8=";
+    })
+  ];
+
+  checkInputs = [ unittestCheckHook ];
+
+  unittestFlagsArray = [ "-s" "test" ];
+
+  pythonImportsCheck = [ "bitstring" ];
+
+  meta = with lib; {
+    description = "Module for binary data manipulation";
+    homepage = "https://github.com/scott-griffiths/bitstring";
+    license = licenses.mit;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ bjornfor ];
+  };
+}

pkgs/python-packages/specific-versions/cryptography/default.nix

@@ -50,7 +50,8 @@ buildPythonPackage rec {
 
   checkInputs = [
     cryptography_vectors
-    hypothesis
+    # Work around `error: infinite recursion encountered`
+    (hypothesis.override { enableDocumentation = false; })
     iso8601
     pretend
     pytest
@@ -58,7 +59,7 @@ buildPythonPackage rec {
   ];
 
   checkPhase = ''
-    py.test --disable-pytest-warnings tests
+    ${pytest}/bin/py.test --disable-pytest-warnings tests
   '';
 
   # IOKit's dependencies are inconsistent between OSX versions, so this is the best we

pkgs/python-packages/specific-versions/twisted.nix

@@ -1,173 +0,0 @@
-{ lib
-, stdenv
-, buildPythonPackage
-, pythonOlder
-, fetchPypi
-, python
-, appdirs
-, attrs
-, automat
-, bcrypt
-, constantly
-, contextvars
-, cryptography
-, git
-, glibcLocales
-, h2
-, hyperlink
-, idna
-, incremental
-, priority
-, pyasn1
-, pyhamcrest
-, pynacl
-, pyopenssl
-, pyserial
-, service-identity
-, setuptools
-, typing-extensions
-, zope_interface
-
-  # for passthru.tests
-, cassandra-driver
-, klein
-, magic-wormhole
-, scrapy
-, treq
-, txaio
-, txamqp
-, txrequests
-, txtorcon
-, thrift
-, nixosTests
-}:
-
-buildPythonPackage rec {
-  pname = "twisted";
-  version = "22.4.0";
-  format = "setuptools";
-
-  disabled = pythonOlder "3.6";
-
-  src = fetchPypi {
-    pname = "Twisted";
-    inherit version;
-    extension = "tar.gz";
-    sha256 = "sha256-oEeZD1ffrh4L0rffJSbU8W3NyEN3TcEIt4xS8qXxNoA=";
-  };
-
-  __darwinAllowLocalNetworking = true;
-
-  propagatedBuildInputs = [
-    attrs
-    automat
-    constantly
-    hyperlink
-    incremental
-    setuptools
-    typing-extensions
-    zope_interface
-  ];
-
-  postPatch = ''
-    echo 'ListingTests.test_localeIndependent.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
-    echo 'ListingTests.test_newFile.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
-    echo 'ListingTests.test_newSingleDigitDayOfMonth.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
-    echo 'ListingTests.test_oldFile.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
-    echo 'ListingTests.test_oldSingleDigitDayOfMonth.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
-
-    echo 'PTYProcessTestsBuilder_AsyncioSelectorReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
-    echo 'PTYProcessTestsBuilder_SelectReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
-
-    echo 'UNIXTestsBuilder_AsyncioSelectorReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
-    echo 'UNIXTestsBuilder_SelectReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
-
-    echo 'FileObserverTests.test_getTimezoneOffsetEastOfUTC.skip = "mktime argument out of range"'>> src/twisted/test/test_log.py
-    echo 'FileObserverTests.test_getTimezoneOffsetWestOfUTC.skip = "mktime argument out of range"'>> src/twisted/test/test_log.py
-    echo 'FileObserverTests.test_getTimezoneOffsetWithoutDaylightSavingTime.skip = "tuple differs, values not"'>> src/twisted/test/test_log.py
-
-    echo 'MulticastTests.test_joinLeave.skip = "No such device"'>> src/twisted/test/test_udp.py
-    echo 'MulticastTests.test_loopback.skip = "No such device"'>> src/twisted/test/test_udp.py
-    echo 'MulticastTests.test_multicast.skip = "Reactor was unclean"'>> src/twisted/test/test_udp.py
-    echo 'MulticastTests.test_multiListen.skip = "No such device"'>> src/twisted/test/test_udp.py
-
-    echo 'DomishExpatStreamTests.test_namespaceWithWhitespace.skip = "syntax error: line 1, column 0"'>> src/twisted/words/test/test_domish.py
-
-    # not packaged
-    substituteInPlace src/twisted/test/test_failure.py \
-      --replace "from cython_test_exception_raiser import raiser  # type: ignore[import]" "raiser = None"
-  '' + lib.optionalString stdenv.isLinux ''
-    echo 'PTYProcessTestsBuilder_EPollReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
-    echo 'PTYProcessTestsBuilder_PollReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
-    echo 'UNIXTestsBuilder_EPollReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
-    echo 'UNIXTestsBuilder_PollReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
-
-    # Patch t.p._inotify to point to libc. Without this,
-    # twisted.python.runtime.platform.supportsINotify() == False
-    substituteInPlace src/twisted/python/_inotify.py --replace \
-      "ctypes.util.find_library(\"c\")" "'${stdenv.cc.libc}/lib/libc.so.6'"
-  '' + lib.optionalString (stdenv.isAarch64 && stdenv.isDarwin) ''
-    echo 'AbortConnectionTests_AsyncioSelectorReactorTests.test_fullWriteBufferAfterByteExchange.skip = "Timeout after 120 seconds"' >> src/twisted/internet/test/test_tcp.py
-    echo 'AbortConnectionTests_AsyncioSelectorReactorTests.test_resumeProducingAbort.skip = "Timeout after 120 seconds"' >> src/twisted/internet/test/test_tcp.py
-  '';
-
-  # Generate Twisted's plug-in cache. Twisted users must do it as well. See
-  # http://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3
-  # and http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=477103 for details.
-  postFixup = ''
-    $out/bin/twistd --help > /dev/null
-  '';
-
-  checkInputs = [
-    git
-    glibcLocales
-    pyhamcrest
-  ]
-  ++ passthru.optional-dependencies.conch
-  # not supported on aarch64-darwin: https://github.com/pyca/pyopenssl/issues/873
-  ++ lib.optionals (!(stdenv.isDarwin && stdenv.isAarch64)) passthru.optional-dependencies.tls;
-
-  checkPhase = ''
-    export SOURCE_DATE_EPOCH=315532800
-    export PATH=$out/bin:$PATH
-    # race conditions when running in paralell
-    ${python.interpreter} -m twisted.trial twisted
-  '';
-
-  passthru = {
-    optional-dependencies = rec {
-      conch = [ appdirs bcrypt cryptography pyasn1 ];
-      conch_nacl = conch ++ [ pynacl ];
-      contextvars = lib.optionals (pythonOlder "3.7") [ contextvars ];
-      http2 = [ h2 priority ];
-      serial = [ pyserial ];
-      tls = [ idna pyopenssl service-identity ];
-    };
-
-    tests = {
-      inherit
-        cassandra-driver
-        klein
-        magic-wormhole
-        scrapy
-        treq
-        txaio
-        txamqp
-        txrequests
-        txtorcon
-        thrift;
-      inherit (nixosTests) buildbot matrix-synapse;
-    };
-  };
-
-  meta = with lib; {
-    homepage = "https://github.com/twisted/twisted";
-    description = "Twisted, an event-driven networking engine written in Python";
-    longDescription = ''
-      Twisted is an event-driven networking engine written in Python
-      and licensed under the MIT license.
-    '';
-    license = licenses.mit;
-    maintainers = with maintainers; [ SuperSandro2000 ];
-  };
-}

pkgs/python-packages/specific-versions/txtorcon.nix

@@ -0,0 +1,48 @@
+{ lib
+, stdenv
+, python
+, buildPythonPackage
+, pythonOlder
+, fetchPypi
+, cryptography
+, incremental
+, twisted
+, automat
+, zope_interface
+, idna
+, pyopenssl
+, service-identity
+, pytestCheckHook
+, mock
+, lsof
+, GeoIP
+}:
+
+buildPythonPackage rec {
+  pname = "txtorcon";
+  version = "22.0.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-iaG2XjKks2nWfmwWY4f7xGjMXQUidEjSOaXn6XGKoFM=";
+  };
+
+  propagatedBuildInputs = [
+    cryptography
+    incremental
+    twisted
+    automat
+    zope_interface
+  ] ++ twisted.optional-dependencies.tls;
+
+  nativeCheckInputs = [ pytestCheckHook mock lsof GeoIP ];
+
+  doCheck = !(stdenv.isDarwin && stdenv.isAarch64);
+
+  meta = {
+    description = "Twisted-based Tor controller client, with state-tracking and configuration abstractions";
+    homepage = "https://github.com/meejah/txtorcon";
+    maintainers = with lib.maintainers; [ jluttine exarkun ];
+    license = lib.licenses.mit;
+  };
+}