use fsspec

leap-stc · Jun 6, 2024 · e56963c · e56963c
1 parent d6437df
commit e56963c
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/feedstock/recipe.py b/feedstock/recipe.py
@@ -162,11 +162,11 @@ async def get_recipe_inputs():
         f"Creating {iid}" >> beam.Create(pattern.items())
         | CheckpointFileTransfer(
             transfer_target=cache_target,
-            max_executors=1,
-            concurrency_per_executor=2,
+            max_executors=10,
+            concurrency_per_executor=3,
             initial_backoff=3.0,  # Try with super long backoff and
             backoff_factor=2.0,
-            fsspec_sync_patch=True,
+            fsspec_sync_patch=False,
         )
         | OpenURLWithFSSpec(cache=None, fsspec_sync_patch=True)
         | OpenWithXarray(xarray_open_kwargs={"use_cftime": True})