@inproceedings{b1e2ec12e6794c0b8afc4ed7f954f938,
title = "A Unified Visual Prompt Tuning Framework with Mixture-of-Experts for Multimodal Information Extraction",
abstract = "Recently, multimodal information extraction has gained increasing attention in social media understanding, as it helps to accomplish the task of information extraction by adding images as auxiliary information to solve the ambiguity problem caused by insufficient semantic information in short texts. Despite their success, current methods do not take full advantage of the information provided by the diverse representations of images. To address this problem, we propose a novel unified visual prompt tuning framework with Mixture-of-Experts to fuse different types of image representations for multimodal information extraction. Extensive experiments conducted on two different multimodal information extraction tasks demonstrate the effectiveness of our method. The source code can be found at https://github.com/xubodhu/VisualPT-MoE.",
keywords = "Mixture-of-Experts, Multimodal information extraction, Prompt learning, Social media",
author = "Bo Xu and Shizhou Huang and Ming Du and Hongya Wang and Hui Song and Yanghua Xiao and Xin Lin",
note = "Publisher Copyright: {\textcopyright} 2023, The Author(s), under exclusive license to Springer Nature Switzerland AG.; 28th International Conference on Database Systems for Advanced Applications, DASFAA 2023 ; Conference date: 17-04-2023 Through 20-04-2023",
year = "2023",
doi = "10.1007/978-3-031-30675-4\_40",
language = "英语",
isbn = "9783031306747",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Science and Business Media Deutschland GmbH",
pages = "544--554",
editor = "Xin Wang and Sapino, \{Maria Luisa\} and Wook-Shin Han and \{El Abbadi\}, Amr and Gill Dobbie and Zhiyong Feng and Yingxiao Shao and Hongzhi Yin",
booktitle = "Database Systems for Advanced Applications - 28th International Conference, DASFAA 2023, Proceedings",
address = "德国",
}