diff --git a/src/nbla/cuda/function/generic/clip_grad_by_norm.cu b/src/nbla/cuda/function/generic/clip_grad_by_norm.cu
index 4bcd23077..cbba94c18 100644
--- a/src/nbla/cuda/function/generic/clip_grad_by_norm.cu
+++ b/src/nbla/cuda/function/generic/clip_grad_by_norm.cu
@@ -31,7 +31,7 @@ __global__ void kernel_clip_grad_by_norm_forward(const int num, T *y,
                                                  const T *x, const T *m,
                                                  const float clip_norm) {
   NBLA_CUDA_KERNEL_LOOP(idx, num) {
-    y[idx] = (T)clip_norm * x[idx] / sqrtf(m[idx]);
+    y[idx] = (T)clip_norm * x[idx] / fmaxf(sqrtf(m[idx]), clip_norm);
   }
 }