diff --git a/mmdet/models/dense_heads/grounding_dino_head.py b/mmdet/models/dense_heads/grounding_dino_head.py
index e9428c0d4de..697981b9651 100644
--- a/mmdet/models/dense_heads/grounding_dino_head.py
+++ b/mmdet/models/dense_heads/grounding_dino_head.py
@@ -551,7 +551,7 @@ def loss_by_feat_single(self, cls_scores: Tensor, bbox_preds: Tensor,
 
         # ===== this change =====
         x_labels = labels.reshape(-1, labels.size(-1))
-        pos_inds = (x_labels > 0).nonzero()
+        pos_inds = (x_labels.sum(1) > 0).nonzero()
         scores = label_weights.new_zeros((x_labels.shape[0],))
         pos_bbox_targets = bbox_targets[pos_inds]
         pos_decode_bbox_targets = bbox_cxcywh_to_xyxy(pos_bbox_targets)
@@ -667,7 +667,7 @@ def _loss_dn_single(self, dn_cls_scores: Tensor, dn_bbox_preds: Tensor,
 
         # ===== this change =====
         x_labels = labels.reshape(-1, labels.size(-1))
-        pos_inds = (x_labels > 0).nonzero()
+        pos_inds = (x_labels.sum(1) > 0).nonzero()
         scores = label_weights.new_zeros((x_labels.shape[0],))
         pos_bbox_targets = bbox_targets[pos_inds]
         pos_decode_bbox_targets = bbox_cxcywh_to_xyxy(pos_bbox_targets)