update GPU stat

matifali · Aug 11, 2023 · b5ece8e · b5ece8e
1 parent 62cf267
commit b5ece8e
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/.github/workflows/deeplearning-nvidia.yaml b/.github/workflows/deeplearning-nvidia.yaml
@@ -24,7 +24,7 @@ jobs:
         run: echo ::set-output name=sha_short::$(git rev-parse --short HEAD)
 
       - name: update coder template
-        uses: matifali/update-coder-template@latest
+        uses: matifali/update-coder-template@v2
         with:
           CODER_URL: ${{ vars.CODER_URL }}
           CODER_TEMPLATE_NAME: "deeplearning-nvidia"

diff --git a/.github/workflows/deeplearning.yaml b/.github/workflows/deeplearning.yaml
@@ -23,7 +23,7 @@ jobs:
         run: echo ::set-output name=sha_short::$(git rev-parse --short HEAD)
 
       - name: update coder template
-        uses: matifali/update-coder-template@latest
+        uses: matifali/update-coder-template@v2
         with:
           CODER_URL: ${{ vars.CODER_URL }}
           CODER_TEMPLATE_NAME: "deeplearning"

diff --git a/matlab/main.tf b/matlab/main.tf
@@ -134,7 +134,7 @@ resource "coder_agent" "main" {
     interval     = 10
     key          = "4_gpu_usage"
     script       = <<EOT
-      nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | awk '{printf "%s%%", $1}'
+      (nvidia-smi 1> /dev/null 2> /dev/null) && (nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | awk '{printf "%s%%", $1}') || echo "N/A"
     EOT
   }
 
@@ -143,7 +143,7 @@ resource "coder_agent" "main" {
     interval     = 10
     key          = "5_gpu_memory_usage"
     script       = <<EOT
-      nvidia-smi --query-gpu=utilization.memory --format=csv,noheader,nounits | awk '{printf "%s%%", $1}'
+      (nvidia-smi 1> /dev/null 2> /dev/null) && (nvidia-smi --query-gpu=utilization.memory --format=csv,noheader,nounits | awk '{printf "%s%%", $1}') || echo "N/A"
     EOT
   }