@inproceedings{wang-etal-2026-llm-assisted,
title = "{LLM}-assisted Data Quality Improvement: {SRL} as a Case Study",
author = "Wang, Lijie and
Liu, Yahui and
Zhang, Yue and
Li, Zhenghua",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1440/",
pages = "30723--30738",
ISBN = "979-8-89176-462-0",
abstract = "Although large language models (LLMs) perform much worse than traditional supervised models on the semantic role labeling (SRL) task, our closer investigation reveals two important findings. First, the existing labeled SRL data is substantially flawed, i.e., of unsatisfactory quality. Second, LLMs can serve as an effective tool for detecting annotation errors and deciding which labels require human review, from the perspective of minimizing human effort. This paper proposes an LLM-assisted data quality improvement approach, and applies it to two SRL evaluation datasets from two different domains. With our approach, human annotators need to review only 11{\%} of the originally annotated arguments flagged for manual inspection, among which 40{\%} are truly erroneous. Meanwhile, our approach suggests 20{\%} new arguments potentially missed in the original labels for human review, among which 60{\%} are indeed missing. In total, our approach locates and corrects 21{\%} errors in the two evaluation datasets. Experiments show that both traditional supervised models and LLMs perform much better on the revised evaluation datasets."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-llm-assisted">
<titleInfo>
<title>LLM-assisted Data Quality Improvement: SRL as a Case Study</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lijie</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yahui</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhenghua</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Although large language models (LLMs) perform much worse than traditional supervised models on the semantic role labeling (SRL) task, our closer investigation reveals two important findings. First, the existing labeled SRL data is substantially flawed, i.e., of unsatisfactory quality. Second, LLMs can serve as an effective tool for detecting annotation errors and deciding which labels require human review, from the perspective of minimizing human effort. This paper proposes an LLM-assisted data quality improvement approach, and applies it to two SRL evaluation datasets from two different domains. With our approach, human annotators need to review only 11% of the originally annotated arguments flagged for manual inspection, among which 40% are truly erroneous. Meanwhile, our approach suggests 20% new arguments potentially missed in the original labels for human review, among which 60% are indeed missing. In total, our approach locates and corrects 21% errors in the two evaluation datasets. Experiments show that both traditional supervised models and LLMs perform much better on the revised evaluation datasets.</abstract>
<identifier type="citekey">wang-etal-2026-llm-assisted</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1440/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>30723</start>
<end>30738</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LLM-assisted Data Quality Improvement: SRL as a Case Study
%A Wang, Lijie
%A Liu, Yahui
%A Zhang, Yue
%A Li, Zhenghua
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wang-etal-2026-llm-assisted
%X Although large language models (LLMs) perform much worse than traditional supervised models on the semantic role labeling (SRL) task, our closer investigation reveals two important findings. First, the existing labeled SRL data is substantially flawed, i.e., of unsatisfactory quality. Second, LLMs can serve as an effective tool for detecting annotation errors and deciding which labels require human review, from the perspective of minimizing human effort. This paper proposes an LLM-assisted data quality improvement approach, and applies it to two SRL evaluation datasets from two different domains. With our approach, human annotators need to review only 11% of the originally annotated arguments flagged for manual inspection, among which 40% are truly erroneous. Meanwhile, our approach suggests 20% new arguments potentially missed in the original labels for human review, among which 60% are indeed missing. In total, our approach locates and corrects 21% errors in the two evaluation datasets. Experiments show that both traditional supervised models and LLMs perform much better on the revised evaluation datasets.
%U https://aclanthology.org/2026.findings-emnlp.1440/
%P 30723-30738
Markdown (Informal)
[LLM-assisted Data Quality Improvement: SRL as a Case Study](https://aclanthology.org/2026.findings-emnlp.1440/) (Wang et al., Findings 2026)
ACL