[train] change all awsv2 to aws (#48255)

awsv2 is not official. should just use aws Signed-off-by: Lonnie Liu <[email protected]>
ray-project · Oct 25, 2024 · 22e266a · 22e266a
1 parent 366a369
commit 22e266a
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 13 deletions.
diff --git a/doc/source/templates/04_finetuning_llms_with_deepspeed/utils.py b/doc/source/templates/04_finetuning_llms_with_deepspeed/utils.py
@@ -12,9 +12,10 @@ def get_hash_from_bucket(
 
     s3_sync_args = s3_sync_args or []
     subprocess.run(
-        ["awsv2", "s3", "cp", "--quiet"]
+        ["aws", "s3", "cp", "--quiet"]
         + s3_sync_args
-        + [os.path.join(bucket_uri, "refs", "main"), "."]
+        + [os.path.join(bucket_uri, "refs", "main"), "."],
+        check=True,
     )
 
     with open(os.path.join(".", "main"), "r") as f:
@@ -70,7 +71,7 @@ def download_model(
     path = get_download_path(model_id)
 
     cmd = (
-        ["awsv2", "s3", "sync"]
+        ["aws", "s3", "sync"]
         + s3_sync_args
         + (["--exclude", "*", "--include", "*token*"] if tokenizer_only else [])
         + [bucket_uri, path]

diff --git a/doc/source/train/examples/deepspeed/gptj_deepspeed_fine_tuning.ipynb b/doc/source/train/examples/deepspeed/gptj_deepspeed_fine_tuning.ipynb
@@ -1406,10 +1406,10 @@
    },
    "outputs": [],
    "source": [
-    "!awsv2 configure set s3.max_concurrent_requests 32\n",
-    "!awsv2 configure set default.s3.preferred_transfer_client crt\n",
-    "!awsv2 configure set default.s3.target_bandwidth 100Gb/s\n",
-    "!awsv2 configure set default.s3.multipart_chunksize 8MB"
+    "!aws configure set s3.max_concurrent_requests 32\n",
+    "!aws configure set default.s3.preferred_transfer_client crt\n",
+    "!aws configure set default.s3.target_bandwidth 100Gb/s\n",
+    "!aws configure set default.s3.multipart_chunksize 8MB"
    ]
   },
   {
@@ -1422,7 +1422,7 @@
    "source": [
     "import os\n",
     "\n",
-    "os.system(f\"awsv2 s3 sync s3://{checkpoint.path} /mnt/local_storage/\")"
+    "os.system(f\"aws s3 sync s3://{checkpoint.path} /mnt/local_storage/\")"
    ]
   },
   {

diff --git a/doc/source/train/examples/lightning/vicuna_13b_lightning_deepspeed_finetune.ipynb b/doc/source/train/examples/lightning/vicuna_13b_lightning_deepspeed_finetune.ipynb
@@ -1075,10 +1075,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!awsv2 configure set s3.max_concurrent_requests 32\n",
-    "!awsv2 configure set default.s3.preferred_transfer_client crt\n",
-    "!awsv2 configure set default.s3.target_bandwidth 100Gb/s\n",
-    "!awsv2 configure set default.s3.multipart_chunksize 8MB"
+    "!aws configure set s3.max_concurrent_requests 32\n",
+    "!aws configure set default.s3.preferred_transfer_client crt\n",
+    "!aws configure set default.s3.target_bandwidth 100Gb/s\n",
+    "!aws configure set default.s3.multipart_chunksize 8MB"
    ]
   },
   {
@@ -1089,7 +1089,7 @@
    "source": [
     "import os\n",
     "\n",
-    "os.system(f\"awsv2 s3 sync s3://{result.checkpoint.path} /mnt/local_storage\")"
+    "os.system(f\"aws s3 sync s3://{result.checkpoint.path} /mnt/local_storage\")"
    ]
   },
   {