Issue using PyTorchYolo

[adrian337]

Hello

I am having difficulty understanding how to correctly use PyTorchYolo

I am first trying to get object detections using the code below:

response = requests.get('https://ultralytics.com/images/zidane.jpg')

img = np.asarray(Image.open(BytesIO(response.content)).resize((416, 416))) 
img_reshape = img.reshape((3, 416, 416))
model_path = "yolov3/PyTorch-YOLOv3/config/yolov3.cfg"
weights_path = "yolov3/PyTorch-YOLOv3/weights/yolov3.weights"
model = load_model(model_path, weights_path)

image = np.stack([img_reshape], axis=0).astype(np.float32) 
x = image.copy()

detector = PyTorchYolo(model=model,
                                  input_shape=(3, 416, 416),
                                  clip_values=(0, 255),
                                  attack_losses=("loss_total",)

dets = detector.predict(x)

This appears to return over 10000 detections - as far as I can see it is not performing Non max suppression.

I was expecting/hoping it to behave like PyTorchFasterRCNN which I have working nicely

Could you please advise on how I should be using the PyTorchYolo class?

Thanks in advance!

[beat-buesser]

Hi @adrian337 I think for now #1796 might be helpful, we are working on improving the documentation.

[adrian337]

Hi @beat-buesser I had seen the issue you mentioned and tried the suggestion, but this does not help at all in this case

[beat-buesser]

I have tried my example code with your image and got for a confidence threshold larger than 0.85 the following prediction:

:
Can you share a complete example including imports etc. that produces your image with predictions? I would be interested to find the reason for the difference.

[adrian337]

Full code I'm using that generated my predictions is:

import cv2
import numpy as np
import matplotlib.pyplot as plt
import requests
from io import BytesIO
from PIL import Image
import torch
from art.estimators.object_detection import PyTorchYolo

# https://github.com/eriklindernoren/PyTorch-YOLOv3
from pytorchyolo.utils.loss import compute_loss
from pytorchyolo.models import load_model
import matplotlib.pyplot as plt
from art.attacks.evasion import ProjectedGradientDescent

COCO_INSTANCE_CATEGORY_NAMES = [
    #"__background__",
    "person",
    "bicycle",
    "car",
    "motorcycle",
    "airplane",
    "bus",
    "train",
    "truck",
    "boat",
    "traffic light",
    "fire hydrant",
    "N/A",
    "stop sign",
    "parking meter",
    "bench",
    "bird",
    "cat",
    "dog",
    "horse",
    "sheep",
    "cow",
    "elephant",
    "bear",
    "zebra",
    "giraffe",
    "N/A",
    "backpack",
    "umbrella",
    "N/A",
    "N/A",
    "handbag",
    "tie",
    "suitcase",
    "frisbee",
    "skis",
    "snowboard",
    "sports ball",
    "kite",
    "baseball bat",
    "baseball glove",
    "skateboard",
    "surfboard",
    "tennis racket",
    "bottle",
    "N/A",
    "wine glass",
    "cup",
    "fork",
    "knife",
    "spoon",
    "bowl",
    "banana",
    "apple",
    "sandwich",
    "orange",
    "broccoli",
    "carrot",
    "hot dog",
    "pizza",
    "donut",
    "cake",
    "chair",
    "couch",
    "potted plant",
    "bed",
    "N/A",
    "dining table",
    "N/A",
    "N/A",
    "toilet",
    "N/A",
    "tv",
    "laptop",
    "mouse",
    "remote",
    "keyboard",
    "cell phone",
    "microwave",
    "oven",
    "toaster",
    "sink",
    "refrigerator",
    "N/A",
    "book",
    "clock",
    "vase",
    "scissors",
    "teddy bear",
    "hair drier",
    "toothbrush",
]

def extract_predictions(predictions_, conf_thresh):
    # Get the predicted class
    predictions_class = [COCO_INSTANCE_CATEGORY_NAMES[i] for i in list(predictions_["labels"])]
  #  print("\npredicted classes:", predictions_class)
    if len(predictions_class) < 1:
        return [],[],[]
    # Get the predicted bounding boxes
    predictions_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(predictions_["boxes"])]

    # Get the predicted prediction score
    predictions_score = list(predictions_["scores"])
   # print("predicted score:", predictions_score)

    # Get a list of index with score greater than threshold
    threshold = conf_thresh 
    predictions_t = [predictions_score.index(x) for x in predictions_score if x > threshold]
    if len(predictions_t) > 0:
        predictions_t = predictions_t#[-1] #indices where score over threshold
    else:
        #no predictions esxceeding threshold
        return [],[],[]
    #predictions in score order
    predictions_boxes =  [predictions_boxes[i] for i in predictions_t]
    predictions_class =  [predictions_class[i] for i in predictions_t]
    predictions_scores =  [predictions_score[i] for i in predictions_t]
    return predictions_class, predictions_boxes, predictions_scores

class YoloV3(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model

    def forward(self, x, targets=None):
        if self.training:
            outputs = self.model(x)
            # loss is averaged over a batch. Thus, for patch generation use batch_size = 1
            loss, loss_components = compute_loss(outputs, targets, self.model)

            loss_components_dict = {}
            loss_components_dict["loss_total"] = loss
            # loss_components_dict["loss_box_reg"] = loss_components[0]
            # loss_components_dict["loss_object"] = loss_components[1]
            # loss_components_dict["loss_classification"] = loss_components[2]
            # loss_components_dict["loss_total"] = loss_components[3]

            return loss_components_dict
        else:
            return self.model(x)


response = requests.get('https://ultralytics.com/images/zidane.jpg')

img = np.asarray(Image.open(BytesIO(response.content)).resize((416, 416))) 
img_reshape = img.reshape((3, 416, 416))
#model = torch.hub.load('ultralytics/yolov3', 'yolov3', pretrained=True)
model_path = "yolov3/PyTorch-YOLOv3/config/yolov3.cfg"
weights_path = "yolov3/PyTorch-YOLOv3/weights/yolov3.weights"
model = load_model(model_path, weights_path)

model = YoloV3(model)

image = np.stack([img_reshape], axis=0).astype(np.float32) 
x = image.copy()



detector = PyTorchYolo(model=model,
                                    device_type='cpu',
                                  input_shape=(3, 416, 416),
                                  clip_values=(0, 255),
                                  attack_losses=("loss_total",)
#                                  attack_losses=("loss_box_reg",
#                                                 "loss_object",
#                                                 "loss_classification",)
#                                                 "loss_total",)
                                  )

dets = detector.predict(x)

preds = extract_predictions(dets[0], 0.1)


def plot_image_with_boxes(img, boxes, pred_cls):
    text_size = 1
    text_th = 1
    rect_th = 1

    for i in range(len(boxes)):
        cv2.rectangle(img, (int(boxes[i][0][0]), int(boxes[i][0][1])), (int(boxes[i][1][0]), int(boxes[i][1][1])), color=(0, 255, 0), thickness=rect_th)
        # Write the prediction class
        cv2.putText(img, pred_cls[i] , (int(boxes[i][0][0]), int(boxes[i][0][1])), cv2.FONT_HERSHEY_SIMPLEX, text_size, (0, 255, 0), thickness=text_th)

    plt.axis("off")
    plt.imshow(img.astype(np.uint8), interpolation="nearest")
    plt.show()

        # Plot predictions
plot_image_with_boxes(img=img, boxes=preds[1], pred_cls=preds[0])

Thanks again!

[beat-buesser]

Hi @adrian337 The bug seems to be in your code at line img_reshape = img.reshape((3, 416, 416)) which results in an input image to the model that looks like:

The operation you are looking for to switch to channels first images is img_reshape = img.transpose((2, 0, 1)) which leads to correct object detections.

[adrian337]

Hi @beat-buesser
Thanks! Fixing this helps me get 'correct' detections.
However it finds many different boxes for the same object (as indeed your example above showed) - should this not be going through non max suppression in the way the PGD does?

Thanks

[beat-buesser]

That's great! The predict method returns the output of the model unchanged to allow the user to define post-processing steps if needed. In this case YOLO creates a large number predicted boxes. PGD is different because it is implemented to create one adversarial example for each benign input sample.