@inproceedings{arraf-etal-2026-beyond,
title = "Beyond Token Noise: Discourse-Level Rewrites Expose Brittleness in {QA} {LLM}s",
author = "Arraf, Mousa and
Guy, Ido and
Radinsky, Kira",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2602/",
pages = "59098--59119",
ISBN = "979-8-89176-459-0",
abstract = "Question answering (QA) systems should remain stable across meaning-preserving variations of the same problem. Earlier robustness work showed brittleness to local lexical perturbations, such as synonym replacement, word swaps, insertions, and deletions. Revisiting this robustness axis for LLMs, we find that high-performing models are now comparatively stable to such edits. However, this stability does not extend to discourse-level rewrites: answer-preserving transformations that alter a question{'}s structure and presentation while preserving its intent and gold answer. Across reasoning-focused QA benchmarks, these rewrites induce substantially larger performance drops than local perturbations. They also change how models fail: using a compact failure taxonomy and Total Variation distance, we show that discourse-level rewrites systematically redistribute error types, revealing distinct failure regimes. Our results suggest that local lexical perturbations remain useful, while discourse-level rewrites provide a complementary and stronger diagnostic signal for evaluating reasoning stability in QA LLMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="arraf-etal-2026-beyond">
<titleInfo>
<title>Beyond Token Noise: Discourse-Level Rewrites Expose Brittleness in QA LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mousa</namePart>
<namePart type="family">Arraf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ido</namePart>
<namePart type="family">Guy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kira</namePart>
<namePart type="family">Radinsky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Question answering (QA) systems should remain stable across meaning-preserving variations of the same problem. Earlier robustness work showed brittleness to local lexical perturbations, such as synonym replacement, word swaps, insertions, and deletions. Revisiting this robustness axis for LLMs, we find that high-performing models are now comparatively stable to such edits. However, this stability does not extend to discourse-level rewrites: answer-preserving transformations that alter a question’s structure and presentation while preserving its intent and gold answer. Across reasoning-focused QA benchmarks, these rewrites induce substantially larger performance drops than local perturbations. They also change how models fail: using a compact failure taxonomy and Total Variation distance, we show that discourse-level rewrites systematically redistribute error types, revealing distinct failure regimes. Our results suggest that local lexical perturbations remain useful, while discourse-level rewrites provide a complementary and stronger diagnostic signal for evaluating reasoning stability in QA LLMs.</abstract>
<identifier type="citekey">arraf-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2602/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>59098</start>
<end>59119</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Token Noise: Discourse-Level Rewrites Expose Brittleness in QA LLMs
%A Arraf, Mousa
%A Guy, Ido
%A Radinsky, Kira
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F arraf-etal-2026-beyond
%X Question answering (QA) systems should remain stable across meaning-preserving variations of the same problem. Earlier robustness work showed brittleness to local lexical perturbations, such as synonym replacement, word swaps, insertions, and deletions. Revisiting this robustness axis for LLMs, we find that high-performing models are now comparatively stable to such edits. However, this stability does not extend to discourse-level rewrites: answer-preserving transformations that alter a question’s structure and presentation while preserving its intent and gold answer. Across reasoning-focused QA benchmarks, these rewrites induce substantially larger performance drops than local perturbations. They also change how models fail: using a compact failure taxonomy and Total Variation distance, we show that discourse-level rewrites systematically redistribute error types, revealing distinct failure regimes. Our results suggest that local lexical perturbations remain useful, while discourse-level rewrites provide a complementary and stronger diagnostic signal for evaluating reasoning stability in QA LLMs.
%U https://aclanthology.org/2026.emnlp-main.2602/
%P 59098-59119
Markdown (Informal)
[Beyond Token Noise: Discourse-Level Rewrites Expose Brittleness in QA LLMs](https://aclanthology.org/2026.emnlp-main.2602/) (Arraf et al., EMNLP 2026)
ACL