Support custom path for installed extensions (#1940)

* feat: the more you fuck around the more you learn
* feat: backwards compatible static file loading
* refactor: update paths for extension files
* refactor: var renaming
* doc: update `LNBITS_EXTENSIONS_PATH` documentation
* fix: default folder install
* feat: install ext without external path
* doc: `PYTHONPATH` no longer required
* fix: add warnings
* fix: missing path
* refactor: re-order statements
* fix: hardcoded path separator

---------

Co-authored-by: dni ⚡ <office@dnilabs.com>

.env.example

@@ -40,6 +40,11 @@ LNBITS_HIDE_API=false
 # GitHub has rate-limits for its APIs. The limit can be increased specifying a GITHUB_TOKEN
 # LNBITS_EXT_GITHUB_TOKEN=github_pat_xxxxxxxxxxxxxxxxxx
 
+# Path where extensions will be installed (defaults to `./lnbits/`).
+# Inside this directory the `extensions` and `upgrades` sub-directories will be created.
+# LNBITS_EXTENSIONS_PATH="/path/to/some/dir"
+
+
 # Extensions to be installed by default. If an extension from this list is uninstalled then it will be re-installed on the next restart.
 # The extension must be removed from this list in order to not be re-installed.
 LNBITS_EXTENSIONS_DEFAULT_INSTALL="tpos"

lnbits/app.py

@@ -9,6 +9,7 @@ import sys
 import traceback
 from hashlib import sha256
 from http import HTTPStatus
+from pathlib import Path
 from typing import Callable, List
 
 from fastapi import FastAPI, HTTPException, Request
@@ -96,6 +97,8 @@ def create_app() -> FastAPI:
     app.add_middleware(InstalledExtensionMiddleware)
     app.add_middleware(ExtensionsRedirectMiddleware)
 
+    register_custom_extensions_path()
+
     # adds security middleware
     add_ip_block_middleware(app)
     add_ratelimit_middleware(app)
@@ -229,9 +232,7 @@ def check_installed_extension_files(ext: InstallableExtension) -> bool:
     if ext.has_installed_version:
         return True
 
-    zip_files = glob.glob(
+    zip_files = glob.glob(os.path.join(settings.lnbits_data_folder, "zips", "*.zip"))
-        os.path.join(settings.lnbits_data_folder, "extensions", "*.zip")
-    )
 
     if f"./{str(ext.zip_path)}" not in zip_files:
         ext.download_archive()
@@ -267,6 +268,25 @@ def register_routes(app: FastAPI) -> None:
             logger.error(f"Could not load extension `{ext.code}`: {str(e)}")
 
 
+def register_custom_extensions_path():
+    if settings.has_default_extension_path:
+        return
+    default_ext_path = os.path.join("lnbits", "extensions")
+    if os.path.isdir(default_ext_path) and len(os.listdir(default_ext_path)) != 0:
+        logger.warning(
+            "You are using a custom extensions path, "
+            + "but the default extensions directory is not empty. "
+            + f"Please clean-up the '{default_ext_path}' directory."
+        )
+        logger.warning(
+            f"You can move the existing '{default_ext_path}' directory to: "
+            + f" '{settings.lnbits_extensions_path}/extensions'"
+        )
+
+    sys.path.append(str(Path(settings.lnbits_extensions_path, "extensions")))
+    sys.path.append(str(Path(settings.lnbits_extensions_path, "upgrades")))
+
+
 def register_new_ext_routes(app: FastAPI) -> Callable:
     # Returns a function that registers new routes for an extension.
     # The returned function encapsulates (creates a closure around)
@@ -303,7 +323,10 @@ def register_ext_routes(app: FastAPI, ext: Extension) -> None:
     if hasattr(ext_module, f"{ext.code}_static_files"):
         ext_statics = getattr(ext_module, f"{ext.code}_static_files")
         for s in ext_statics:
-            app.mount(s["path"], s["app"], s["name"])
+            static_dir = Path(
+                settings.lnbits_extensions_path, "extensions", *s["path"].split("/")
+            )
+            app.mount(s["path"], StaticFiles(directory=static_dir), s["name"])
 
     if hasattr(ext_module, f"{ext.code}_redirect_paths"):
         ext_redirects = getattr(ext_module, f"{ext.code}_redirect_paths")

lnbits/extension_manager.py

@@ -182,12 +182,19 @@ class Extension(NamedTuple):
     upgrade_hash: Optional[str] = ""
 
     @property
-    def module_name(self):
+    def module_name(self) -> str:
-        return (
+        if self.is_upgrade_extension:
-            f"lnbits.extensions.{self.code}"
+            if settings.has_default_extension_path:
-            if self.upgrade_hash == ""
+                return f"lnbits.upgrades.{self.code}-{self.upgrade_hash}"
-            else f"lnbits.upgrades.{self.code}-{self.upgrade_hash}.{self.code}"
+            return f"{self.code}-{self.upgrade_hash}"
-        )
+
+        if settings.has_default_extension_path:
+            return f"lnbits.extensions.{self.code}"
+        return self.code
+
+    @property
+    def is_upgrade_extension(self) -> bool:
+        return self.upgrade_hash != ""
 
     @classmethod
     def from_installable_ext(cls, ext_info: "InstallableExtension") -> "Extension":
@@ -205,7 +212,7 @@ class Extension(NamedTuple):
 
 class ExtensionManager:
     def __init__(self) -> None:
-        p = Path(settings.lnbits_path, "extensions")
+        p = Path(settings.lnbits_extensions_path, "extensions")
         Path(p).mkdir(parents=True, exist_ok=True)
         self._extension_folders: List[Path] = [f for f in p.iterdir() if f.is_dir()]
 
@@ -330,21 +337,25 @@ class InstallableExtension(BaseModel):
 
     @property
     def zip_path(self) -> Path:
-        extensions_data_dir = Path(settings.lnbits_data_folder, "extensions")
+        extensions_data_dir = Path(settings.lnbits_data_folder, "zips")
         Path(extensions_data_dir).mkdir(parents=True, exist_ok=True)
         return Path(extensions_data_dir, f"{self.id}.zip")
 
     @property
     def ext_dir(self) -> Path:
-        return Path(settings.lnbits_path, "extensions", self.id)
+        return Path(settings.lnbits_extensions_path, "extensions", self.id)
 
     @property
     def ext_upgrade_dir(self) -> Path:
-        return Path("lnbits", "upgrades", f"{self.id}-{self.hash}")
+        return Path(
+            settings.lnbits_extensions_path, "upgrades", f"{self.id}-{self.hash}"
+        )
 
     @property
     def module_name(self) -> str:
+        if settings.has_default_extension_path:
             return f"lnbits.extensions.{self.id}"
+        return self.id
 
     @property
     def module_installed(self) -> bool:
@@ -389,21 +400,26 @@ class InstallableExtension(BaseModel):
 
     def extract_archive(self):
         logger.info(f"Extracting extension {self.name} ({self.installed_version}).")
-        Path("lnbits", "upgrades").mkdir(parents=True, exist_ok=True)
+        Path(settings.lnbits_extensions_path, "upgrades").mkdir(
-        shutil.rmtree(self.ext_upgrade_dir, True)
+            parents=True, exist_ok=True
-        with zipfile.ZipFile(self.zip_path, "r") as zip_ref:
-            zip_ref.extractall(self.ext_upgrade_dir)
-        generated_dir_name = os.listdir(self.ext_upgrade_dir)[0]
-        os.rename(
-            Path(self.ext_upgrade_dir, generated_dir_name),
-            Path(self.ext_upgrade_dir, self.id),
         )
 
+        tmp_dir = Path(settings.lnbits_data_folder, "unzip-temp", self.hash)
+        shutil.rmtree(tmp_dir, True)
+
+        with zipfile.ZipFile(self.zip_path, "r") as zip_ref:
+            zip_ref.extractall(tmp_dir)
+        generated_dir_name = os.listdir(tmp_dir)[0]
+        shutil.rmtree(self.ext_upgrade_dir, True)
+        shutil.copytree(
+            Path(tmp_dir, generated_dir_name),
+            Path(self.ext_upgrade_dir),
+        )
+        shutil.rmtree(tmp_dir, True)
+
         # Pre-packed extensions can be upgraded
         # Mark the extension as installed so we know it is not the pre-packed version
-        with open(
+        with open(Path(self.ext_upgrade_dir, "config.json"), "r+") as json_file:
-            Path(self.ext_upgrade_dir, self.id, "config.json"), "r+"
-        ) as json_file:
             config_json = json.load(json_file)
 
             self.name = config_json.get("name")
@@ -419,10 +435,7 @@ class InstallableExtension(BaseModel):
                 )
 
         shutil.rmtree(self.ext_dir, True)
-        shutil.copytree(
+        shutil.copytree(Path(self.ext_upgrade_dir), Path(self.ext_dir))
-            Path(self.ext_upgrade_dir, self.id),
-            Path(settings.lnbits_path, "extensions", self.id),
-        )
         logger.success(f"Extension {self.name} ({self.installed_version}) installed.")
 
     def nofiy_upgrade(self) -> None:

lnbits/helpers.py

@@ -30,6 +30,10 @@ def url_for(endpoint: str, external: Optional[bool] = False, **params: Any) -> s
 def template_renderer(additional_folders: Optional[List] = None) -> Jinja2Templates:
     folders = ["lnbits/templates", "lnbits/core/templates"]
     if additional_folders:
+        additional_folders += [
+            Path(settings.lnbits_extensions_path, "extensions", f)
+            for f in additional_folders
+        ]
         folders.extend(additional_folders)
     t = Jinja2Templates(loader=jinja2.FileSystemLoader(folders))
 

lnbits/server.py

@@ -38,8 +38,10 @@ def main(
     # create data dir if it does not exist
     Path(settings.lnbits_data_folder).mkdir(parents=True, exist_ok=True)
 
-    # create extension dir if it does not exist
+    # create `extensions`` dir if it does not exist
-    Path(settings.lnbits_path, "extensions").mkdir(parents=True, exist_ok=True)
+    Path(settings.lnbits_extensions_path, "extensions").mkdir(
+        parents=True, exist_ok=True
+    )
 
     set_cli_settings(host=host, port=port, forwarded_allow_ips=forwarded_allow_ips)
 

lnbits/settings.py

@@ -275,10 +275,15 @@ class EnvSettings(LNbitsSettings):
     forwarded_allow_ips: str = Field(default="*")
     lnbits_title: str = Field(default="LNbits API")
     lnbits_path: str = Field(default=".")
+    lnbits_extensions_path: str = Field(default="lnbits")
     lnbits_commit: str = Field(default="unknown")
     super_user: str = Field(default="")
     version: str = Field(default="0.0.0")
 
+    @property
+    def has_default_extension_path(self) -> bool:
+        return self.lnbits_extensions_path == "lnbits"
+
 
 class SaaSSettings(LNbitsSettings):
     lnbits_saas_callback: Optional[str] = Field(default=None)