update a2mim release

configs/openmixup/pretrain/_base_/models/simmim/convnext_b.py

@@ -0,0 +1,15 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='MIMConvNeXt',
+        arch="base",
+        out_indices=(3,),  # x-1: stage-x
+        act_cfg=dict(type='GELU'),
+        drop_path_rate=0.0,
+        gap_before_final_norm=False,
+        replace=False,  # use residual mask token
+        mask_layer=0, mask_token='learnable',
+    ),
+    neck=dict(type='SimMIMNeck', in_channels=1024, encoder_stride=32),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/convnext_s.py

@@ -0,0 +1,15 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='MIMConvNeXt',
+        arch="small",
+        out_indices=(3,),  # x-1: stage-x
+        act_cfg=dict(type='GELU'),
+        drop_path_rate=0.0,
+        gap_before_final_norm=False,
+        replace=False,  # use residual mask token
+        mask_layer=0, mask_token='learnable',
+    ),
+    neck=dict(type='SimMIMNeck', in_channels=768, encoder_stride=32),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/convnext_t.py

@@ -0,0 +1,15 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='MIMConvNeXt',
+        arch="tiny",
+        out_indices=(3,),  # x-1: stage-x
+        act_cfg=dict(type='GELU'),
+        drop_path_rate=0.0,
+        gap_before_final_norm=False,
+        replace=False,  # use residual mask token
+        mask_layer=0, mask_token='learnable',
+    ),
+    neck=dict(type='SimMIMNeck', in_channels=768, encoder_stride=32),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/r101.py

@@ -0,0 +1,13 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='MIMResNet',
+        depth=101,
+        mask_layer=0, mask_token='learnable',
+        num_stages=4,
+        out_indices=(3,),  # no conv-1, x-1: stage-x
+        norm_cfg=dict(type='SyncBN'),
+        style='pytorch'),
+    neck=dict(type='SimMIMNeck', in_channels=2048, encoder_stride=32),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/r152.py

@@ -0,0 +1,13 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='MIMResNet',
+        depth=152,
+        mask_layer=0, mask_token='learnable',
+        num_stages=4,
+        out_indices=(3,),  # no conv-1, x-1: stage-x
+        norm_cfg=dict(type='SyncBN'),
+        style='pytorch'),
+    neck=dict(type='SimMIMNeck', in_channels=2048, encoder_stride=32),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/r200.py

@@ -0,0 +1,13 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='MIMResNet',
+        depth=200,
+        mask_layer=0, mask_token='learnable',
+        num_stages=4,
+        out_indices=(3,),  # no conv-1, x-1: stage-x
+        norm_cfg=dict(type='SyncBN'),
+        style='pytorch'),
+    neck=dict(type='SimMIMNeck', in_channels=2048, encoder_stride=32),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/vit_large.py

@@ -0,0 +1,14 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='SimMIMViT',
+        arch='large',
+        replace=True,
+        mask_layer=0, mask_token='learnable',
+        img_size=224,
+        drop_rate=0., drop_path_rate=0.1,
+        use_window=True, init_values=0.1,  # SimMIM: use init_value and relative pos encoding
+    ),
+    neck=dict(type='SimMIMNeck', in_channels=1024, encoder_stride=16),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/_base_/models/simmim/vit_small.py

@@ -0,0 +1,14 @@
+# model settings
+model = dict(
+    type='SimMIM',
+    backbone=dict(
+        type='SimMIMViT',
+        arch='small',
+        replace=True,
+        mask_layer=0, mask_token='learnable',
+        img_size=224,
+        drop_rate=0., drop_path_rate=0.1,
+        use_window=True, init_values=0.1,  # SimMIM: use init_value and relative pos encoding
+    ),
+    neck=dict(type='SimMIMNeck', in_channels=768, encoder_stride=16),
+    head=dict(type='SimMIMHead', encoder_in_channels=3))

configs/openmixup/pretrain/a2mim/imagenet/convnext_b_l3_sz224_init_8xb256_cos_ep300.py

@@ -0,0 +1,69 @@
+_base_ = [
+    '../../_base_/models/a2mim/convnext_b.py',
+    '../../_base_/datasets/imagenet/a2mim_rgb_m_sz224_rrc08_bs64.py',
+    '../../_base_/default_runtime.py',
+]
+
+# model settings
+model = dict(
+    backbone=dict(
+        mask_layer=3, mask_token="learnable",
+        mask_init=1e-6,  # init residual gamma
+    ),
+    head=dict(
+        fft_weight=0., fft_focal=False,
+    ),
+)
+
+# dataset
+data = dict(
+    imgs_per_gpu=256, workers_per_gpu=10,
+    train=dict(
+        feature_mode=None, feature_args=dict(),
+        mask_pipeline=[
+            dict(type='BlockwiseMaskGenerator',
+                input_size=224, mask_patch_size=32, mask_ratio=0.6, model_patch_size=32,  # stage 3
+                mask_color='mean', mask_only=False),
+        ],
+))
+
+# interval for accumulate gradient
+update_interval = 1  # bs256 x 8gpus = bs2048
+
+# additional hooks
+custom_hooks = [
+    dict(type='SAVEHook',
+        save_interval=626 * 25,  # plot every 25 ep
+        iter_per_epoch=626),
+]
+
+# optimizer
+optimizer = dict(
+    type='AdamW',
+    lr=3e-4 * 2048 / 512,  # 3e-4 * 4 for bs2048
+    betas=(0.9, 0.999), weight_decay=0.05, eps=1e-8,
+    paramwise_options={
+        '(bn|ln|gn)(\d+)?.(weight|bias)': dict(weight_decay=0.),
+        'bias': dict(weight_decay=0.),
+        'gamma': dict(weight_decay=0.),
+        'mask_token': dict(weight_decay=0., lr_mult=1e-1,),
+        'mask_gamma': dict(weight_decay=0., lr_mult=1e-1,),
+    })
+
+# fp16
+use_fp16 = True
+fp16 = dict(type='mmcv', loss_scale='dynamic')
+# optimizer args
+optimizer_config = dict(update_interval=update_interval)
+
+# lr scheduler
+lr_config = dict(
+    policy='StepFixCosineAnnealing',
+    by_epoch=False, min_lr=1e-5,
+    warmup='linear',
+    warmup_iters=10, warmup_by_epoch=True,
+    warmup_ratio=1e-6,
+)
+
+# runtime settings
+runner = dict(type='EpochBasedRunner', max_epochs=300)

configs/openmixup/pretrain/a2mim/imagenet/convnext_t_l3_sz224_init_8xb256_cos_ep300.py

@@ -22,7 +22,7 @@
         feature_mode=None, feature_args=dict(),
         mask_pipeline=[
             dict(type='BlockwiseMaskGenerator',
-                input_size=224, mask_patch_size=32, mask_ratio=0.6, model_patch_size=32,  # stage 3 in MogaNet
+                input_size=224, mask_patch_size=32, mask_ratio=0.6, model_patch_size=32,  # stage 3
                 mask_color='mean', mask_only=False),
         ],
 ))
@@ -33,14 +33,14 @@
 # additional hooks
 custom_hooks = [
     dict(type='SAVEHook',
-        save_interval=626 * 10,  # plot every 10 ep
+        save_interval=626 * 25,  # plot every 25 ep
         iter_per_epoch=626),
 ]
 
 # optimizer
 optimizer = dict(
     type='AdamW',
-    lr=1e-4 * 2048 / 512,  # 4e-3 for bs2048
+    lr=3e-4 * 2048 / 512,  # 3e-4 * 4 for bs2048
     betas=(0.9, 0.999), weight_decay=0.05, eps=1e-8,
     paramwise_options={
         '(bn|ln|gn)(\d+)?.(weight|bias)': dict(weight_decay=0.),

configs/openmixup/pretrain/a2mim/imagenet/moganet_s_l3_sz224_init_8xb256_cos_ep300.py

@@ -17,13 +17,12 @@
 
 # dataset
 data = dict(
-    # imgs_per_gpu=256, workers_per_gpu=10,
-    imgs_per_gpu=64, workers_per_gpu=4,
+    imgs_per_gpu=256, workers_per_gpu=10,
     train=dict(
         feature_mode=None, feature_args=dict(),
         mask_pipeline=[
             dict(type='BlockwiseMaskGenerator',
-                input_size=224, mask_patch_size=32, mask_ratio=0.6, model_patch_size=32,  # stage 3 in MogaNet
+                input_size=224, mask_patch_size=32, mask_ratio=0.6, model_patch_size=32,  # stage 3
                 mask_color='mean', mask_only=False),
         ],
 ))
@@ -34,26 +33,25 @@
 # additional hooks
 custom_hooks = [
     dict(type='SAVEHook',
-        save_interval=626 * 10,  # plot every 10 ep
+        save_interval=626 * 25,  # plot every 25 ep
         iter_per_epoch=626),
 ]
 
 # optimizer
 optimizer = dict(
     type='AdamW',
-    lr=1e-4 * 2048 / 512,  # 4e-3 for bs2048
+    lr=3e-4 * 2048 / 512,  # 3e-4 * 4 for bs2048
     betas=(0.9, 0.999), weight_decay=0.05, eps=1e-8,
     paramwise_options={
         '(bn|ln|gn)(\d+)?.(weight|bias)': dict(weight_decay=0.),
         'bias': dict(weight_decay=0.),
-        'layer_scale': dict(weight_decay=0.),
-        'scale': dict(weight_decay=0.),
+        'gamma': dict(weight_decay=0.),
         'mask_token': dict(weight_decay=0., lr_mult=1e-1,),
         'mask_gamma': dict(weight_decay=0., lr_mult=1e-1,),
     })
 
 # fp16
-use_fp16 = False
+use_fp16 = True
 fp16 = dict(type='mmcv', loss_scale='dynamic')
 # optimizer args
 optimizer_config = dict(update_interval=update_interval)

configs/openmixup/pretrain/a2mim/imagenet/r200_l3_sz224_init_8xb256_cos_ep300.py

@@ -0,0 +1,63 @@
+_base_ = [
+    '../../_base_/models/a2mim/r200.py',
+    '../../_base_/datasets/imagenet/a2mim_rgb_m_sz224_rrc08_bs64.py',
+    '../../_base_/default_runtime.py',
+]
+
+# model settings
+model = dict(
+    backbone=dict(
+        mask_layer=3, mask_token="learnable",
+        mask_init=1e-6,  # init residual gamma
+))
+
+# dataset
+data = dict(
+    imgs_per_gpu=256, workers_per_gpu=10,
+    train=dict(
+        feature_mode=None, feature_args=dict(),
+        mask_pipeline=[
+            dict(type='BlockwiseMaskGenerator',
+                input_size=224, mask_patch_size=32, mask_ratio=0.6, model_patch_size=16,  # stage 3
+                mask_color='mean', mask_only=False),
+        ],
+))
+
+# interval for accumulate gradient
+update_interval = 1
+
+# additional hooks
+custom_hooks = [
+    dict(type='SAVEHook',
+        save_interval=626 * 10,  # plot every 10 ep
+        iter_per_epoch=626),
+]
+
+# optimizer
+optimizer = dict(
+    type='AdamW',
+    lr=3e-4 * 2048 / 512,  # 1.2e-3 for bs2048
+    betas=(0.9, 0.999), weight_decay=0.05, eps=1e-8,
+    paramwise_options={
+        '(bn|ln|gn)(\d+)?.(weight|bias)': dict(weight_decay=0.),
+        'bias': dict(weight_decay=0.),
+        'mask_token': dict(weight_decay=0., lr_mult=1e-1,),
+    })
+
+# fp16
+use_fp16 = False
+fp16 = dict(type='mmcv', loss_scale='dynamic')
+# optimizer args
+optimizer_config = dict(update_interval=update_interval)
+
+# lr scheduler
+lr_config = dict(
+    policy='StepFixCosineAnnealing',
+    by_epoch=False, min_lr=1e-5,
+    warmup='linear',
+    warmup_iters=10, warmup_by_epoch=True,
+    warmup_ratio=1e-6,
+)
+
+# runtime settings
+runner = dict(type='EpochBasedRunner', max_epochs=300)

configs/openmixup/pretrain/a2mim/imagenet/r50_l3_sz224_init_8xb256_cos_ep300.py

@@ -29,7 +29,7 @@
 # additional hooks
 custom_hooks = [
     dict(type='SAVEHook',
-        save_interval=626 * 10,  # plot every 10 ep
+        save_interval=626 * 25,  # plot every 25 ep
         iter_per_epoch=626),
 ]
 

configs/openmixup/pretrain/a2mim/imagenet/vit_base_l0_sz224_8xb128_accu2_step_ep800.py

@@ -12,7 +12,7 @@
 
 # dataset
 data = dict(
-    imgs_per_gpu=128, workers_per_gpu=10,
+    imgs_per_gpu=128, workers_per_gpu=12,
     train=dict(
         feature_mode=None, feature_args=dict(),
         mask_pipeline=[
@@ -28,7 +28,7 @@
 # additional hooks
 custom_hooks = [
     dict(type='SAVEHook',
-        save_interval=1252 * 10,  # plot every 10 ep
+        save_interval=1252 * 25,  # plot every 25 ep
         iter_per_epoch=1252),
 ]
 

configs/openmixup/pretrain/a2mim/imagenet/vit_large_l0_sz224_8xb128_accu2_step_ep800.py

@@ -12,7 +12,7 @@
 
 # dataset
 data = dict(
-    imgs_per_gpu=128, workers_per_gpu=10,
+    imgs_per_gpu=128, workers_per_gpu=12,
     train=dict(
         feature_mode=None, feature_args=dict(),
         mask_pipeline=[
@@ -28,7 +28,7 @@
 # additional hooks
 custom_hooks = [
     dict(type='SAVEHook',
-        save_interval=1252 * 10,  # plot every 10 ep
+        save_interval=1252 * 25,  # plot every 25 ep
         iter_per_epoch=1252),
 ]