diff --git a/keras_cv/layers/preprocessing/random_gaussian_blur.py b/keras_cv/layers/preprocessing/random_gaussian_blur.py
index 7b8d151302..a2209f8bd5 100644
--- a/keras_cv/layers/preprocessing/random_gaussian_blur.py
+++ b/keras_cv/layers/preprocessing/random_gaussian_blur.py
@@ -61,7 +61,9 @@ def __init__(self, kernel_size, factor, **kwargs):
                 )
 
     def get_random_transformation(self, **kwargs):
-        factor = self.factor()
+        # `factor` must not become too small otherwise numerical issues occur.
+        # keras.backend.epsilon() behaves like 0 without causing `nan`s
+        factor = tf.math.maximum(self.factor(), keras.backend.epsilon())
         blur_v = RandomGaussianBlur.get_kernel(factor, self.y)
         blur_h = RandomGaussianBlur.get_kernel(factor, self.x)
         blur_v = tf.reshape(blur_v, [self.y, 1, 1, 1])