feat: Accept multiple paths in the vfolder mkdir API

changes/1803.feature.md

@@ -0,0 +1 @@
+Add support for multi directory mkdir by fixing cli to accept multiple arguments and by adding list type annotation to accept multiple directories

src/ai/backend/client/cli/vfolder.py

@@ -430,7 +430,7 @@ def cp(filenames):
 
 @vfolder.command()
 @click.argument("name", type=str)
-@click.argument("path", type=str)
+@click.argument("paths", type=str, nargs=-1)
 @click.option(
     "-p",
     "--parents",
@@ -445,17 +445,19 @@ def cp(filenames):
     is_flag=True,
     help="Skip an error caused by file not found",
 )
-def mkdir(name, path, parents, exist_ok):
+def mkdir(name, paths, parents, exist_ok):
     """Create an empty directory in the virtual folder.
 
     \b
     NAME: Name of a virtual folder.
-    PATH: The name or path of directory. Parent directories are created automatically
+    PATH: Mutliple arguments of name or path of directory seperated by space. Parent directories are created automatically
           if they do not exist.
+
+    Example: backend.ai vfolder mkdir my_vfolder "dir1" "dir2" "dir3"
     """
     with Session() as session:
         try:
-            session.VFolder(name).mkdir(path, parents=parents, exist_ok=exist_ok)
+            session.VFolder(name).mkdir(paths, parents=parents, exist_ok=exist_ok)
             print_done("Done.")
         except Exception as e:
             print_error(e)

src/ai/backend/client/func/vfolder.py

@@ -1,6 +1,6 @@
 import asyncio
 from pathlib import Path
-from typing import Dict, Mapping, Optional, Sequence, Union
+from typing import Dict, List, Mapping, Optional, Sequence, Union
 
 import aiohttp
 import janus
@@ -475,7 +475,7 @@ async def _upload_recursively(
             if path.is_file():
                 file_list.append(path)
             else:
-                await self._mkdir(path.relative_to(base_path))
+                await self._mkdir([path.relative_to(base_path)])
                 dir_list.append(path)
         await self._upload_files(file_list, basedir, dst_dir, chunk_size, address_map)
         for dir in dir_list:
@@ -506,7 +506,7 @@ async def upload(
 
     async def _mkdir(
         self,
-        path: Union[str, Path],
+        path: str | Path | List[str | Path],
         parents: Optional[bool] = False,
         exist_ok: Optional[bool] = False,
     ) -> str:
@@ -522,7 +522,7 @@ async def _mkdir(
     @api_function
     async def mkdir(
         self,
-        path: Union[str, Path],
+        path: str | Path | List[str | Path],
         parents: Optional[bool] = False,
         exist_ok: Optional[bool] = False,
     ) -> str:

src/ai/backend/common/validators.py

@@ -245,7 +245,11 @@ def __init__(
         self._relative_only = relative_only
 
     def check_and_return(self, value: Any) -> _PurePath:
-        p = _PurePath(value)
+        try:
+            p = _PurePath(value)
+        except (TypeError, ValueError):
+            self._failure("cannot parse value as a path", value=value)
+
         if self._relative_only and p.is_absolute():
             self._failure("expected relative path but the value is absolute", value=value)
         if self._base_path is not None:

src/ai/backend/manager/api/vfolder.py

@@ -1228,7 +1228,7 @@ async def update_vfolder_options(
 @vfolder_permission_required(VFolderPermission.READ_WRITE)
 @check_api_params(
     t.Dict({
-        t.Key("path"): t.String,
+        t.Key("path"): t.String | t.List(t.String),
         t.Key("parents", default=True): t.ToBool,
         t.Key("exist_ok", default=False): t.ToBool,
     })
@@ -1239,7 +1239,7 @@ async def mkdir(request: web.Request, params: Any, row: VFolderRow) -> web.Respo
     folder_name = request.match_info["name"]
     access_key = request["keypair"]["access_key"]
     log.info(
-        "VFOLDER.MKDIR (email:{}, ak:{}, vf:{}, path:{})",
+        "VFOLDER.MKDIR (email:{}, ak:{}, vf:{}, paths:{})",
         request["user"]["email"],
         access_key,
         folder_name,

src/ai/backend/manager/models/vfolder.py

@@ -753,7 +753,7 @@ async def prepare_vfolder_mounts(
                 params={
                     "volume": storage_manager.split_host(vfolder["host"])[1],
                     "vfid": str(VFolderID(vfolder["quota_scope_id"], vfolder["id"])),
-                    "relpath": str(user_scope.user_uuid.hex),
+                    "relpaths": [str(user_scope.user_uuid.hex)],
                     "exist_ok": True,
                 },
             ):

src/ai/backend/storage/api/manager.py

@@ -4,6 +4,8 @@
 
 from __future__ import annotations
 
+import asyncio
+import errno
 import json
 import logging
 import os
@@ -755,7 +757,7 @@ async def mkdir(request: web.Request) -> web.Response:
     class Params(TypedDict):
         volume: str
         vfid: VFolderID
-        relpath: PurePosixPath
+        relpath: PurePosixPath | List[PurePosixPath]
         parents: bool
         exist_ok: bool
 
@@ -767,7 +769,8 @@ class Params(TypedDict):
                 {
                     t.Key("volume"): t.String(),
                     t.Key("vfid"): tx.VFolderID(),
-                    t.Key("relpath"): tx.PurePath(relative_only=True),
+                    t.Key("relpath"): tx.PurePath(relative_only=True)
+                    | t.List(tx.PurePath(relative_only=True)),
                     t.Key("parents", default=True): t.ToBool,
                     t.Key("exist_ok", default=False): t.ToBool,
                 },
@@ -776,14 +779,54 @@ class Params(TypedDict):
     ) as params:
         await log_manager_api_entry(log, "mkdir", params)
         ctx: RootContext = request.app["ctx"]
+        vfid = params["vfid"]
+        parents = params["parents"]
+        exist_ok = params["exist_ok"]
+        relpath = params["relpath"]
+        relpaths = relpath if isinstance(relpath, list) else [relpath]
+        error_file_list: List[str] = []
+        success_file_list: List[str] = []
+
         async with ctx.get_volume(params["volume"]) as volume:
-            with handle_fs_errors(volume, params["vfid"]):
-                await volume.mkdir(
-                    params["vfid"],
-                    params["relpath"],
-                    parents=params["parents"],
-                    exist_ok=params["exist_ok"],
+            mkdir_tasks = [
+                volume.mkdir(vfid, rpath, parents=parents, exist_ok=exist_ok) for rpath in relpaths
+            ]
+            result_group = await asyncio.gather(*mkdir_tasks, return_exceptions=True)
+
+            # if result group is all false throw error.
+            if all([isinstance(res, FileExistsError) for res in result_group]):
+                raise web.HTTPBadRequest(
+                    body=json.dumps(
+                        {
+                            "msg": "FileExistsError None of directories created because they already exist",
+                            "errno": errno.EEXIST,
+                        },
+                    ),
+                    content_type="application/json",
                 )
+
+            if any([isinstance(res, BaseException) for res in result_group]):
+                for relpath, result_or_exception in zip(relpaths, result_group):
+                    if isinstance(result_or_exception, BaseException):
+                        log.error(
+                            "{}",
+                            repr(result_or_exception),
+                            exc_info=result_or_exception,
+                        )
+                        error_file_list.append(f"[Errno {errno.EEXIST}] File exists: {relpath}")
+                    else:
+                        success_file_list.append(str(relpath))
+
+                raise web.HTTPBadRequest(
+                    body=json.dumps(
+                        {
+                            "error_file_list": error_file_list,
+                            "success_file_list": success_file_list,
+                        },
+                    ),
+                    content_type="application/json",
+                )
+
         return web.Response(status=204)