@inproceedings{bc0ba6faf4a7492b8d9108f52b761916,
title = "SemInfer: Accelerating LLM-Based Semantic Data Processing via Sparse Indexing",
abstract = "Integrating LLMs for data processing enables semantic querying but causes GPU memory bottlenecks and redundant computations. We present SemInfer, an acceleration system for batch semantic processing that treats the KV Cache as a semantic index, offloading pre-computed caches to host storage to eliminate redundancy. To reduce the index size, we propose a pruning strategy based on last-layer aggregated attention to accurately retain critical semantic tokens. Furthermore, we employ a pipeline mechanism to enable the asynchronous overlapping of CPU-GPU transmission and inference computation. This demonstration showcases the complete workflow of SemInfer on the IMDB dataset, achieving up to a 16.3x inference speedup over direct LLM inference and a 90\% reduction in index size with few semantic accuracy loss.",
keywords = "Data Processing, KV Cache Pruning, LLM",
author = "Wentao Zhu and Yuyun Zhang and Huiqi Hu and Peng Cai and Xuan Zhou and Yaoqiang Xu and Chen Qian",
note = "Publisher Copyright: {\textcopyright} The Author(s), under exclusive license to Springer Nature Singapore Pte Ltd. 2026.; 31st International Conference on Database Systems for Advanced Applications, DASFAA 2026 ; Conference date: 27-04-2026 Through 30-04-2026",
year = "2026",
doi = "10.1007/978-981-92-0378-9\_51",
language = "英语",
isbn = "9789819203772",
series = "Lecture Notes in Computer Science",
publisher = "Springer Science and Business Media Deutschland GmbH",
pages = "703--707",
editor = "Hyungsoo Jung and Tianzheng Wang and Masashi Toyoda and Hyuk-Yoon Kwon and Jae-woong Lee",
booktitle = "Database Systems for Advanced Applications - 31st International Conference, DASFAA 2026, Proceedings",
address = "德国",
}