@inproceedings{de38da3f352d44a3afdc16b4ef8b229c,
title = "VioLET: Vision-Language Efficient Tuning with Collaborative Multi-modal Gradients",
abstract = "Parameter-Efficient Tuning (PET) has emerged as a leading advancement in both Natural Language Processing and Computer Vision, enabling efficient accommodation of downstream tasks without costly fine-tuning. However, most existing PET approaches are limited to uni-modal tuning, even for vision-language models like CLIP. We investigate this limitation and demonstrate that simultaneous tuning of the two modalities in such models leads to multi-modal forgetting and catastrophic performance degradation, particularly when generalizing to new classes. To address this issue, we propose a novel PET approach called VioLET (Vision Language Efficient Tuning) that utilizes collaborative multi-modal gradients to unlock the full potential of both modalities. Specifically, we incorporate an additional visual encoder without learnable parameters and use these two visual encoders to compute the gradients of the context parameters separately. When conflicts arise, we replace the original gradient with an orthogonal gradient. Extensive experiments are conducted on few-shot recognition and unseen class generalization tasks using ResNet-50 or ViT/B-16 as the backbone. VioLET consistently outperforms several state-of-the-art methods on 11 datasets, showcasing its superiority over existing PET approaches. The code is available at https://github.com/Wang-Yaoming/VioLET.",
keywords = "few-shot recognition, multi-modal, parameter efficient tuning, prompt learning, vision language",
author = "Yaoming Wang and Yuchen Liu and Xiaopeng Zhang and Jin Li and Bowen Shi and Chenglin Li and Wenrui Dai and Hongkai Xiong and Qi Tian",
note = "Publisher Copyright: {\textcopyright} 2023 ACM.; 31st ACM International Conference on Multimedia, MM 2023 ; Conference date: 29-10-2023 Through 03-11-2023",
year = "2023",
month = oct,
day = "27",
doi = "10.1145/3581783.3611706",
language = "英语",
series = "MM 2023 - Proceedings of the 31st ACM International Conference on Multimedia",
publisher = "Association for Computing Machinery, Inc",
pages = "4595--4605",
booktitle = "MM 2023 - Proceedings of the 31st ACM International Conference on Multimedia",
}