add docs

dlt/common/storages/fsspec_filesystem.py

@@ -6,6 +6,7 @@
 import pathlib
 import posixpath
 from io import BytesIO
+from gzip import GzipFile
 from typing import (
     Literal,
     cast,
@@ -43,6 +44,7 @@ class FileItem(TypedDict, total=False):
 
     file_url: str
     file_name: str
+    relative_path: str
     mime_type: str
     encoding: Optional[str]
     modification_date: pendulum.DateTime
@@ -197,7 +199,7 @@ def open(  # noqa: A003
         mode: str = "rb",
         compression: Literal["auto", "disable", "enable"] = "auto",
         **kwargs: Any,
-    ) -> IO[Any]:
+    ) -> Union[GzipFile, IO[Any]]:
         """Open the file as a fsspec file.
 
         This method opens the file represented by this dictionary as a file-like object using

docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md

@@ -296,7 +296,8 @@ data. You can quickly build pipelines to:
 #### `FileItem` Fields:
 
 - `file_url` - Complete URL of the file; also the primary key (e.g. `file://`).
-- `file_name` - Name or relative path of the file from the bucket URL.
+- `file_name` - Name of the file from the bucket URL.
+- `relative_path` - Relative path of the file in the bucket.
 - `mime_type` - File's mime type; sourced from the bucket provider or inferred from its extension.
 - `modification_date` - File's last modification time (format: `pendulum.DateTime`).
 - `size_in_bytes` - File size.

tests/common/storages/utils.py

@@ -49,6 +49,7 @@ def assert_sample_files(
         "gzip/taxi.csv.gz",
         "sample.txt",
     }
+    expected_file_names = [path.split("/")[-1] for path in minimally_expected_file_items]
     assert len(all_file_items) == len(minimally_expected_file_items)
 
     for item in all_file_items:
@@ -58,6 +59,7 @@ def assert_sample_files(
         # is valid url
         file_url_parsed = urlparse(item["file_url"])
         assert isinstance(item["file_name"], str)
+        assert item["file_name"] in expected_file_names
         assert file_url_parsed.path.endswith(item["file_name"])
         assert item["file_url"].startswith(config.protocol)
         assert isinstance(item["mime_type"], str)