@inproceedings{87253751bfe34c0187a28f726155342f,
title = "DocAsRef: An Empirical Study on Repurposing Reference-based Summary Quality Metrics as Reference-free Metrics",
abstract = "Automated summary quality assessment falls into two categories: reference-based and reference-free. Reference-based metrics, historically deemed more accurate due to the additional information provided by human-written references, are limited by their reliance on human input. In this paper, we hypothesize that the comparison methodologies used by some reference-based metrics to evaluate a system summary against its corresponding reference can be effectively adapted to assess it against its source document, thereby transforming these metrics into reference-free ones. Experimental results support this hypothesis. After being repurposed reference-freely, the zero-shot BERTScore using the pretrained DeBERTa-large-MNLI model of <0.5B parameters consistently outperforms its original reference-based version across various aspects on the SummEval and Newsroom datasets. It also excels in comparison to most existing reference-free metrics and closely competes with zero-shot summary evaluators based on GPT-3.5.",
author = "Bao, \{Forrest Sheng\} and Ruixuan Tu and Ge Luo and Yinfei Yang and Hebi Li and Minghui Qiu and Youbiao He and Cen Chen",
note = "Publisher Copyright: {\textcopyright} 2023 Association for Computational Linguistics.; 2023 Findings of the Association for Computational Linguistics: EMNLP 2023 ; Conference date: 06-12-2023 Through 10-12-2023",
year = "2023",
doi = "10.18653/v1/2023.findings-emnlp.87",
language = "英语",
series = "Findings of the Association for Computational Linguistics: EMNLP 2023",
publisher = "Association for Computational Linguistics (ACL)",
pages = "1226--1235",
booktitle = "Findings of the Association for Computational Linguistics",
address = "澳大利亚",
}