-
Notifications
You must be signed in to change notification settings - Fork 0
/
CLIP-L_k400_32x4x1.py
240 lines (240 loc) · 8.01 KB
/
CLIP-L_k400_32x4x1.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
default_scope = 'mmaction'
default_hooks = dict(
runtime_info=dict(type='RuntimeInfoHook'),
timer=dict(type='IterTimerHook'),
logger=dict(type='LoggerHook', interval=20, ignore_last=False),
param_scheduler=dict(type='ParamSchedulerHook'),
checkpoint=dict(
type='CheckpointHook', interval=1, max_keep_ckpts=5, save_best='auto'),
sampler_seed=dict(type='DistSamplerSeedHook'),
sync_buffers=dict(type='SyncBuffersHook'))
env_cfg = dict(
cudnn_benchmark=False,
mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
dist_cfg=dict(backend='nccl'))
log_processor = dict(type='LogProcessor', window_size=20, by_epoch=True)
vis_backends = [dict(type='LocalVisBackend')]
visualizer = dict(
type='ActionVisualizer', vis_backends=[dict(type='LocalVisBackend')])
log_level = 'INFO'
load_from = None
resume = False
model = dict(
type='Recognizer3D',
backbone=dict(
type='ViT_Zero_CLIP',
pretrained='your_clip_path',
input_resolution=224,
patch_size=14,
num_frames=32,
width=1024,
layers=24,
heads=16,
dropout_rate=0.1,
adapter_scale=0.5,
num_tadapter=2,
stdha_cfg=dict(shift_div=8, divide_head=False, long_shift_div=8)),
cls_head=dict(
type='I3DHead',
in_channels=1024,
num_classes=400,
spatial_type='avg',
dropout_ratio=0.5,
average_clips='prob'),
data_preprocessor=dict(
type='ActionDataPreprocessor',
mean=[122.769, 116.74, 104.04],
std=[68.493, 66.63, 70.321],
format_shape='NCTHW'))
dataset_type = 'VideoDataset'
data_prefix = 'your_path_k400/'
data_root = 'your_path_k400/kinetics_400_320_30fps_train'
data_root_val = 'your_path_k400/kinetics_400_320_30fps_val'
ann_file_train = 'data/kinetics400/kinetics400_train_list_videos.txt'
ann_file_val = 'data/kinetics400/kinetics400_val_list_videos.txt'
ann_file_test = 'data/kinetics400/kinetics400_val_list_videos.txt'
file_client_args = dict(io_backend='ceph')
train_pipeline = [
dict(type='DecordInit', io_backend='ceph'),
dict(type='SampleFrames', clip_len=32, frame_interval=4, num_clips=1),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(
type='PytorchVideoWrapper',
op='RandAugment',
magnitude=7,
num_layers=4),
dict(type='RandomResizedCrop'),
dict(type='Resize', scale=(224, 224), keep_ratio=False),
dict(type='Flip', flip_ratio=0.5),
dict(type='RandomErasing', erase_prob=0.25, mode='rand'),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]
val_pipeline = [
dict(type='DecordInit', io_backend='ceph'),
dict(
type='SampleFrames',
clip_len=32,
frame_interval=4,
num_clips=1,
test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(type='CenterCrop', crop_size=224),
dict(type='Flip', flip_ratio=0),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]
test_pipeline = [
dict(type='DecordInit', io_backend='ceph'),
dict(
type='SampleFrames',
clip_len=32,
frame_interval=4,
num_clips=3,
test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 224)),
dict(type='CenterCrop', crop_size=224),
dict(type='Flip', flip_ratio=0),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]
train_dataloader = dict(
batch_size=2,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=True),
dataset=dict(
type='VideoDataset',
ann_file='data/kinetics400/kinetics400_train_list_videos.txt',
data_prefix=dict(
video=
'your_path_k400/kinetics_400_320_30fps_train'
),
pipeline=[
dict(type='DecordInit', io_backend='ceph'),
dict(
type='SampleFrames',
clip_len=32,
frame_interval=4,
num_clips=1),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(
type='PytorchVideoWrapper',
op='RandAugment',
magnitude=7,
num_layers=4),
dict(type='RandomResizedCrop'),
dict(type='Resize', scale=(224, 224), keep_ratio=False),
dict(type='Flip', flip_ratio=0.5),
dict(type='RandomErasing', erase_prob=0.25, mode='rand'),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]))
val_dataloader = dict(
batch_size=2,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=False),
dataset=dict(
type='VideoDataset',
ann_file='data/kinetics400/kinetics400_val_list_videos.txt',
data_prefix=dict(
video=
'your_path_k400/kinetics_400_320_30fps_val'
),
pipeline=[
dict(type='DecordInit', io_backend='ceph'),
dict(
type='SampleFrames',
clip_len=32,
frame_interval=4,
num_clips=1,
test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(type='CenterCrop', crop_size=224),
dict(type='Flip', flip_ratio=0),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
],
test_mode=True))
test_dataloader = dict(
batch_size=1,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=False),
dataset=dict(
type='VideoDataset',
ann_file='data/kinetics400/kinetics400_val_list_videos.txt',
data_prefix=dict(
video=
'your_path_k400/kinetics_400_320_30fps_val'
),
pipeline=[
dict(type='DecordInit', io_backend='ceph'),
dict(
type='SampleFrames',
clip_len=32,
frame_interval=4,
num_clips=3,
test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 224)),
dict(type='CenterCrop', crop_size=224),
dict(type='Flip', flip_ratio=0),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
],
test_mode=True))
val_evaluator = dict(type='AccMetric')
test_evaluator = dict(type='AccMetric')
train_cfg = dict(
type='EpochBasedTrainLoop',
max_epochs=40,
val_begin=1,
dynamic_intervals=[(1, 5), (25, 1)])
val_cfg = dict(type='ValLoop')
test_cfg = dict(type='TestLoop')
accumulative_counts = 2
auto_scale_lr = dict(enable=True, base_batch_size=32)
optim_wrapper = dict(
type='GradMonitorAmpOptimWrapper',
optimizer=dict(
type='AdamW', lr=0.0003, betas=(0.9, 0.999), weight_decay=0.05),
constructor='GradMonitorSwinOptimWrapperConstructor',
accumulative_counts=2,
paramwise_cfg=dict(
class_embedding=dict(decay_mult=0.0),
positional_embedding=dict(decay_mult=0.0),
temporal_embedding=dict(decay_mult=0.0),
absolute_pos_embed=dict(decay_mult=0.0),
ln_1=dict(decay_mult=0.0),
ln_2=dict(decay_mult=0.0),
ln_pre=dict(decay_mult=0.0),
ln_post=dict(decay_mult=0.0),
backbone=dict(lr_mult=0.1)))
param_scheduler = [
dict(
type='LinearLR',
start_factor=0.1,
by_epoch=True,
begin=0,
end=2.5,
convert_to_iter_based=True),
dict(
type='CosineAnnealingLR',
T_max=40,
eta_min=0,
by_epoch=True,
begin=0,
end=40)
]
find_unused_parameters = True
launcher = 'slurm'
work_dir = '.work_dirs/recognition/vit_zero_clip/N006cc_vit_zero_large_clip_k400_32x4x1_div8_40e_long8'
randomness = dict(seed=None, diff_rank_seed=False, deterministic=False)
dist_params = dict(port=21592)