Merge pull request #94 from NeurodataWithoutBorders/additional-url-re…

…solver

Additional url resolver needed by dendro

lindi/LindiRemfile/LindiRemfile.py

@@ -2,6 +2,7 @@
 import time
 import os
 import requests
+from .additional_url_resolvers import get_additional_url_resolvers
 from ..LocalCache.LocalCache import LocalCache
 
 default_min_chunk_size = 128 * 1024  # This is different from Remfile - this is an important decision because it determines the chunk size in the LocalCache
@@ -41,6 +42,7 @@ def __init__(
             Does not support using multiple threads
             Does not use memory cache if LocalCache is specified
             Handles DANDI authentication
+            Handles additional_url_resolver
 
         A note:
             In the context of LINDI, this LindiRemfile is going to be used for loading
@@ -367,6 +369,8 @@ def _resolve_dandi_url(url: str):
 
 
 def _resolve_url(url: str):
+    for aur in get_additional_url_resolvers():
+        url = aur(url)
     if url in _global_resolved_urls:
         elapsed = time.time() - _global_resolved_urls[url]["timestamp"]
         if elapsed < 60 * 10:

lindi/LindiRemfile/additional_url_resolvers.py

@@ -0,0 +1,14 @@
+from typing import Callable, List
+
+
+_global = {
+    'additional_url_resolvers': []
+}
+
+
+def get_additional_url_resolvers() -> List[Callable[[str], str]]:
+    return _global['additional_url_resolvers']
+
+
+def add_additional_url_resolver(resolver: Callable[[str], str]):
+    _global['additional_url_resolvers'].append(resolver)

lindi/__init__.py

@@ -3,3 +3,4 @@
 from .LindiStagingStore import LindiStagingStore, StagingArea  # noqa: F401
 from .LocalCache.LocalCache import LocalCache, ChunkTooLargeError  # noqa: F401
 from .File.File import File  # noqa: F401
+from .LindiRemfile.additional_url_resolvers import add_additional_url_resolver  # noqa: F401

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lindi"
-version = "0.4.0a1"
+version = "0.4.0a2"
 description = ""
 authors = [
     "Jeremy Magland <[email protected]>",