即插即用！清华国科大等推出视觉微调框架，仅需调整5%骨干网络参数 CVPR2025

import torch.nn asnnimporttorch.nn.functionalasF# ------------------------------Mona模块------------------------------INNER_DIM = 64class MonaOp(nn.Module):def __init__(self, in_features):super().__init__()self.conv1 = nn.Conv2d(in_features, in_features, kernel_size=3, padding=3 // 2, groups=in_features)self.conv2 = nn.Conv2d(in_features, in_features, kernel_size=5, padding=5 // 2, groups=in_features)self.conv3 = nn.Conv2d(in_features, in_features, kernel_size=7, padding=7 // 2, groups=in_features)self.projector = nn.Conv2d(in_features, in_features, kernel_size=1, )def forward(self, x):identity = xconv1_x = self.conv1(x)conv2_x = self.conv2(x)conv3_x = self.conv3(x)x = (conv1_x + conv2_x + conv3_x) / 3.0 + identityidentity = xx = self.projector(x)return identity + xclass Mona(BaseModule):def __init__(self,in_dim,factor=4):super().__init__()self.project1 = nn.Linear(in_dim, INNER_DIM)self.nonlinear = F.geluself.project2 = nn.Linear(INNER_DIM, in_dim)self.dropout = nn.Dropout(p=0.1)self.adapter_conv = MonaOp(INNER_DIM)self.norm = nn.LayerNorm(in_dim)self.gamma = nn.Parameter(torch.ones(in_dim) * 1e-6)self.gammax = nn.Parameter(torch.ones(in_dim))def forward(self, x, hw_shapes=None):identity = xx = self.norm(x) * self.gamma + x * self.gammaxproject1 = self.project1(x)b, n, c = project1.shapeh, w = hw_shapesproject1 = project1.reshape(b, h, w, c).permute(0, 3, 1, 2)project1 = self.adapter_conv(project1)project1 = project1.permute(0, 2, 3, 1).reshape(b, n, c)nonlinear = self.nonlinear(project1)nonlinear = self.dropout(nonlinear)project2 = self.project2(nonlinear)return identity + project2#------------------------------插入模式------------------------------#此处省略部分 Swin组件实现，仅提供 Mona插入模式。class SwinBlock(BaseModule):""""Args:embed_dims (int): The feature dimension.num_heads (int): Parallel attention heads.feedforward_channels (int): The hidden dimension for FFNs.window_size (int, optional): The local window scale. Default: 7.shift (bool, optional): whether to shift window or not. Default False.qkv_bias (bool, optional): enable bias for qkv if True. Default: True.qk_scale (float | None, optional): Override default qk scale ofhead_dim ** -0.5 if set. Default: None.drop_rate (float, optional): Dropout rate. Default: 0.attn_drop_rate (float, optional): Attention dropout rate. Default: 0.drop_path_rate (float, optional): Stochastic depth rate. Default: 0.act_cfg (dict, optional): The config dict of activation function.Default: dict(type='GELU').norm_cfg (dict, optional): The config dict of normalization.Default: dict(type='LN').with_cp (bool, optional): Use checkpoint or not. Using checkpointwill save some memory while slowing down the training speed.Default: False.init_cfg (dict | list | None, optional): The init config.Default: None."""def __init__(self,embed_dims,num_heads,feedforward_channels,window_size=7,shift=False,qkv_bias=True,qk_scale=None,drop_rate=0.,attn_drop_rate=0.,drop_path_rate=0.,act_cfg=dict(type='GELU'),norm_cfg=dict(type='LN'),with_cp=False,init_cfg=None):super(SwinBlock, self).__init__()self.init_cfg = init_cfgself.with_cp = with_cpself.norm1 = build_norm_layer(norm_cfg, embed_dims)[1]self.attn = ShiftWindowMSA(embed_dims=embed_dims,num_heads=num_heads,window_size=window_size,shift_size=window_size // 2 if shift else 0,qkv_bias=qkv_bias,qk_scale=qk_scale,attn_drop_rate=attn_drop_rate,proj_drop_rate=drop_rate,dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate),init_cfg=None)self.norm2 = build_norm_layer(norm_cfg, embed_dims)[1]self.ffn = FFN(embed_dims=embed_dims,feedforward_channels=feedforward_channels,num_fcs=2,ffn_drop=drop_rate,dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate),act_cfg=act_cfg,add_identity=True,init_cfg=None)self.mona1 = Mona(embed_dims, 8)self.mona2 = Mona(embed_dims, 8)def forward(self, x, hw_shape):def _inner_forward(x):identity = xx = self.norm1(x)x = self.attn(x, hw_shape)x = x + identityx = self.mona1(x, hw_shape)identity = xx = self.norm2(x)x = self.ffn(x, identity=identity)x = self.mona2(x, hw_shape)return xif self.with_cp and x.requires_grad:x = cp.checkpoint(_inner_forward, x)else:x = _inner_forward(x)return x

@misc{yin20245100breakingperformanceshackles,      title={5%>100%: Breaking Performance Shackles of Full Fine-Tuning on Visual Recognition Tasks},       author={Dongshuo Yin and Leiyi Hu and Bin Li and Youqun Zhang and Xue Yang},      year={2024},      eprint={2408.08345},      archivePrefix={arXiv},      primaryClass={cs.CV},      url={https://arxiv.org/abs/2408.08345}, }