From ce644ac46f123b5f3d8f069d478f7648919a9f62 Mon Sep 17 00:00:00 2001 From: huanghaian Date: Tue, 31 Oct 2023 19:52:02 +0800 Subject: [PATCH] fix bug --- ...ss_swin-t_a_fpn_dyhead_pretrain_odinw13.py | 31 +++++++++---- ...ss_swin-t_a_fpn_dyhead_pretrain_odinw35.py | 43 +++++++++++++++++-- mmdet/datasets/base_det_dataset.py | 3 +- mmdet/datasets/dataset_wrappers.py | 5 --- .../evaluator/multi_datasets_evaluator.py | 39 +++-------------- 5 files changed, 70 insertions(+), 51 deletions(-) diff --git a/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw13.py b/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw13.py index 1f3915f5ff6..57ac02ae2fc 100644 --- a/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw13.py +++ b/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw13.py @@ -13,6 +13,7 @@ data_root=_data_root, ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), + test_mode=True, pipeline=_base_.test_pipeline, return_classes=True) val_evaluator_AerialMaritimeDrone = dict( @@ -39,6 +40,7 @@ data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, caption_prompt=caption_prompt, + test_mode=True, return_classes=True) val_evaluator_Aquarium = dict( type='CocoMetric', @@ -46,7 +48,7 @@ metric='bbox') # ---------------------3 CottontailRabbits---------------------# -class_name = ('Cottontail-Rabbit',) +class_name = ('Cottontail-Rabbit', ) metainfo = dict(classes=class_name) _data_root = data_root + 'CottontailRabbits/' dataset_CottontailRabbits = dict( @@ -56,6 +58,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_CottontailRabbits = dict( type='CocoMetric', @@ -63,7 +66,7 @@ metric='bbox') # ---------------------4 EgoHands---------------------# -class_name = ('hand',) +class_name = ('hand', ) metainfo = dict(classes=class_name) _data_root = data_root + 'EgoHands/generic/' dataset_EgoHands = dict( @@ -73,6 +76,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_EgoHands = dict( type='CocoMetric', @@ -90,6 +94,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_NorthAmericaMushrooms = dict( type='CocoMetric', @@ -97,7 +102,7 @@ metric='bbox') # ---------------------6 Packages---------------------# -class_name = ('package',) +class_name = ('package', ) metainfo = dict(classes=class_name) _data_root = data_root + 'Packages/Raw/' dataset_Packages = dict( @@ -107,6 +112,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_Packages = dict( type='CocoMetric', @@ -127,6 +133,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_PascalVOC = dict( type='CocoMetric', @@ -134,24 +141,25 @@ metric='bbox') # ---------------------8 pistols---------------------# -class_name = ('pistol',) +class_name = ('pistol', ) metainfo = dict(classes=class_name) _data_root = data_root + 'pistols/export/' dataset_pistols = dict( type=dataset_type, metainfo=metainfo, data_root=_data_root, - ann_file='valid/annotations_without_background.json', - data_prefix=dict(img='valid/'), + ann_file='val_annotations_without_background.json', + data_prefix=dict(img=''), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_pistols = dict( type='CocoMetric', - ann_file=_data_root + 'valid/annotations_without_background.json', + ann_file=_data_root + 'val_annotations_without_background.json', metric='bbox') # ---------------------9 pothole---------------------# -class_name = ('pothole',) +class_name = ('pothole', ) metainfo = dict(classes=class_name) _data_root = data_root + 'pothole/' dataset_pothole = dict( @@ -161,6 +169,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_pothole = dict( type='CocoMetric', @@ -168,7 +177,7 @@ metric='bbox') # ---------------------10 Raccoon---------------------# -class_name = ('raccoon',) +class_name = ('raccoon', ) metainfo = dict(classes=class_name) _data_root = data_root + 'Raccoon/Raccoon.v2-raw.coco/' dataset_Raccoon = dict( @@ -178,6 +187,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_Raccoon = dict( type='CocoMetric', @@ -195,6 +205,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_ShellfishOpenImages = dict( type='CocoMetric', @@ -212,6 +223,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_thermalDogsAndPeople = dict( type='CocoMetric', @@ -229,6 +241,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_VehiclesOpenImages = dict( type='CocoMetric', diff --git a/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw35.py b/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw35.py index c361811d40b..66669d5be5b 100644 --- a/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw35.py +++ b/configs/odinw/glip_atss_swin-t_a_fpn_dyhead_pretrain_odinw35.py @@ -14,6 +14,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_AerialMaritimeDrone_large = dict( type='CocoMetric', @@ -31,6 +32,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_AerialMaritimeDrone_tiled = dict( type='CocoMetric', @@ -41,7 +43,7 @@ class_name = ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z') metainfo = dict(classes=class_name) -_data_root = data_root + 'AmericanSignLanguageLetters/American Sign Language Letters.v1-v1.coco/' # noqa +_data_root = data_root + 'AmericanSignLanguageLetters/American Sign Language Letters.v1-v1.coco/' # noqa dataset_AmericanSignLanguageLetters = dict( type=dataset_type, metainfo=metainfo, @@ -49,6 +51,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_AmericanSignLanguageLetters = dict( type='CocoMetric', @@ -67,6 +70,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_Aquarium = dict( type='CocoMetric', @@ -84,6 +88,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_BCCD = dict( type='CocoMetric', @@ -103,6 +108,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_boggleBoards = dict( type='CocoMetric', @@ -120,6 +126,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_brackishUnderwater = dict( type='CocoMetric', @@ -139,6 +146,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_ChessPieces = dict( type='CocoMetric', @@ -156,6 +164,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_CottontailRabbits = dict( type='CocoMetric', @@ -173,6 +182,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_dice = dict( type='CocoMetric', @@ -191,6 +201,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_DroneControl = dict( type='CocoMetric', @@ -208,6 +219,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_EgoHands_generic = dict( type='CocoMetric', @@ -225,6 +237,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_EgoHands_specific = dict( type='CocoMetric', @@ -242,6 +255,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_HardHatWorkers = dict( type='CocoMetric', @@ -259,6 +273,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_MaskWearing = dict( type='CocoMetric', @@ -276,6 +291,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_MountainDewCommercial = dict( type='CocoMetric', @@ -293,6 +309,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_NorthAmericaMushrooms = dict( type='CocoMetric', @@ -319,6 +336,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_openPoetryVision = dict( type='CocoMetric', @@ -349,6 +367,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_OxfordPets_by_breed = dict( type='CocoMetric', @@ -366,6 +385,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_OxfordPets_by_species = dict( type='CocoMetric', @@ -383,6 +403,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_PKLot = dict( type='CocoMetric', @@ -400,6 +421,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_Packages = dict( type='CocoMetric', @@ -420,6 +442,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_PascalVOC = dict( type='CocoMetric', @@ -434,13 +457,14 @@ type=dataset_type, metainfo=metainfo, data_root=_data_root, - ann_file='valid/annotations_without_background.json', - data_prefix=dict(img='valid/'), + ann_file='val_annotations_without_background.json', + data_prefix=dict(img=''), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_pistols = dict( type='CocoMetric', - ann_file=_data_root + 'valid/annotations_without_background.json', + ann_file=_data_root + 'val_annotations_without_background.json', metric='bbox') # ---------------------25 plantdoc---------------------# @@ -466,6 +490,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_plantdoc = dict( type='CocoMetric', @@ -483,6 +508,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_pothole = dict( type='CocoMetric', @@ -500,6 +526,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_Raccoon = dict( type='CocoMetric', @@ -520,6 +547,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_selfdrivingCar = dict( type='CocoMetric', @@ -537,6 +565,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_ShellfishOpenImages = dict( type='CocoMetric', @@ -554,6 +583,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_ThermalCheetah = dict( type='CocoMetric', @@ -571,6 +601,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_thermalDogsAndPeople = dict( type='CocoMetric', @@ -589,6 +620,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_UnoCards = dict( type='CocoMetric', @@ -606,6 +638,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_VehiclesOpenImages = dict( type='CocoMetric', @@ -623,6 +656,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_WildfireSmoke = dict( type='CocoMetric', @@ -641,6 +675,7 @@ ann_file='valid/annotations_without_background.json', data_prefix=dict(img='valid/'), pipeline=_base_.test_pipeline, + test_mode=True, return_classes=True) val_evaluator_websiteScreenshots = dict( type='CocoMetric', diff --git a/mmdet/datasets/base_det_dataset.py b/mmdet/datasets/base_det_dataset.py index 7cc39aadb32..8b3876d5c06 100644 --- a/mmdet/datasets/base_det_dataset.py +++ b/mmdet/datasets/base_det_dataset.py @@ -40,7 +40,8 @@ def __init__(self, self.return_classes = return_classes self.caption_prompt = caption_prompt if self.caption_prompt is not None: - assert self.return_classes, 'return_classes must be True when using caption_prompt' + assert self.return_classes, \ + 'return_classes must be True when using caption_prompt' if file_client_args is not None: raise RuntimeError( 'The `file_client_args` is deprecated, ' diff --git a/mmdet/datasets/dataset_wrappers.py b/mmdet/datasets/dataset_wrappers.py index 4a16ac828bd..d4e26e07c0f 100644 --- a/mmdet/datasets/dataset_wrappers.py +++ b/mmdet/datasets/dataset_wrappers.py @@ -258,8 +258,3 @@ def __init__(self, def get_dataset_source(self, idx: int) -> int: dataset_idx, _ = self._get_ori_dataset_idx(idx) return dataset_idx - - def __getitem__(self, idx): - data_info = super().__getitem__(idx) - data_info['data_samples'].dataset_idx = self.get_dataset_source(idx) - return data_info diff --git a/mmdet/evaluation/evaluator/multi_datasets_evaluator.py b/mmdet/evaluation/evaluator/multi_datasets_evaluator.py index 54daa83d721..05262e9ce77 100644 --- a/mmdet/evaluation/evaluator/multi_datasets_evaluator.py +++ b/mmdet/evaluation/evaluator/multi_datasets_evaluator.py @@ -1,16 +1,13 @@ # Copyright (c) OpenMMLab. All rights reserved. -import bisect import warnings from collections import OrderedDict -from typing import Any, Optional, Sequence, Union +from typing import Sequence, Union -import numpy as np from mmengine.dist import (broadcast_object_list, collect_results, is_main_process) from mmengine.evaluator import BaseMetric, Evaluator from mmengine.evaluator.metric import _to_cpu from mmengine.registry import EVALUATOR -from mmengine.structures import BaseDataElement from mmdet.utils import ConfigType @@ -50,28 +47,6 @@ def _get_cumulative_sizes(self): dataset_slices = self.dataset_meta['cumulative_sizes'] return dataset_slices - def process(self, - data_samples: Sequence[BaseDataElement], - data_batch: Optional[Any] = None): - """Convert ``BaseDataSample`` to dict and invoke process method of each - metric. - - Args: - data_samples (Sequence[BaseDataElement]): predictions of the model, - and the ground truth of the validation set. - data_batch (Any, optional): A batch of data from the dataloader. - """ - dataset_slices = self._get_cumulative_sizes() - assert len(dataset_slices) == len(self.dataset_prefixes) - - for data, data_sample in zip(data_batch, data_samples): - dataset_idx = data_sample.dataset_idx - if isinstance(data_sample, BaseDataElement): - self.metrics[dataset_idx].process([data], - [data_sample.to_dict()]) - else: - self.metrics[dataset_idx].process([data], [data_sample]) - def evaluate(self, size: int) -> dict: """Invoke ``evaluate`` method of each metric and collect the metrics dictionary. @@ -91,23 +66,23 @@ def evaluate(self, size: int) -> dict: dataset_slices = self._get_cumulative_sizes() assert len(dataset_slices) == len(self.dataset_prefixes) - dataset_slices.insert(0, 0) - dataset_slices = np.diff(dataset_slices).tolist() - for dataset_prefix, dataset_slice, metric in zip( - self.dataset_prefixes, dataset_slices, self.metrics): + for dataset_prefix, start, end, metric in zip( + self.dataset_prefixes, [0] + dataset_slices[:-1], + dataset_slices, self.metrics): if len(metric.results) == 0: warnings.warn( f'{metric.__class__.__name__} got empty `self.results`.' 'Please ensure that the processed results are properly ' 'added into `self.results` in `process` method.') - results = collect_results(metric.results, dataset_slice, + results = collect_results(metric.results, size, metric.collect_device) if is_main_process(): # cast all tensors in results list to cpu results = _to_cpu(results) - _metrics = metric.compute_metrics(results) + _metrics = metric.compute_metrics( + results[start:end]) # type: ignore if metric.prefix: final_prefix = '/'.join((dataset_prefix, metric.prefix))