From fa81e251be9c3cb1ef64ca2f454f4143961b1fa8 Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Tue, 23 Apr 2024 23:06:35 +0200 Subject: [PATCH] Automatically create folders for local filesystem --- dlt/destinations/impl/filesystem/filesystem.py | 8 ++++++++ tests/load/pipeline/test_filesystem_pipeline.py | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dlt/destinations/impl/filesystem/filesystem.py b/dlt/destinations/impl/filesystem/filesystem.py index 5dae4bf295..5871b8d67c 100644 --- a/dlt/destinations/impl/filesystem/filesystem.py +++ b/dlt/destinations/impl/filesystem/filesystem.py @@ -73,7 +73,15 @@ def __init__( load_package_timestamp=dlt.current.load_package()["state"]["created_at"], # type: ignore extra_placeholders=config.extra_placeholders, ) + + # We would like to avoid failing for local filesystem where + # deeply nested directory will not exist before writing a file. + # It `auto_mkdir` is disabled by default in fsspec so we made some + # trade offs between different options and decided on this. item = self.make_remote_path() + if self.config.protocol.startswith("file"): + fs_client.makedirs(posixpath.dirname(item), exist_ok=True) + fs_client.put_file(local_path, item) def make_remote_path(self) -> str: diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py index e25e2b9f0a..353f9d81d2 100644 --- a/tests/load/pipeline/test_filesystem_pipeline.py +++ b/tests/load/pipeline/test_filesystem_pipeline.py @@ -305,14 +305,9 @@ def count(*args, **kwargs) -> Any: "hiphip": counter("Hurraaaa"), } now = pendulum.now() - kwargs = {} - if default_buckets_env.startswith("file://"): - kwargs = {"auto_mkdir": True} - fs_destination = filesystem( layout=layout, extra_placeholders=extra_placeholders, - kwargs=kwargs, current_datetime=counter(now), ) pipeline = dlt.pipeline(