@inproceedings{rao-layek-2026-answer,
title = "What Does Answer Change Rate Actually Measure? A Specificity Audit of Causal Intervention Tests for Explanation Faithfulness",
author = "Rao, Durgesh and
Layek, Dr. Shirshendu",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.35/",
pages = "441--452",
ISBN = "979-8-89176-478-1",
abstract = "A growing line of work tests whether large language model (LLM) explanations are faithful by intervening on the explanation (removing, perturbing, or swapping it) and checking whether the model{'}s final answer changes {--} an Answer Change Rate (ACR), read as causal evidence. We argue this inference requires a control existing work has not consistently reported: whether the answer changes comparably under an unrelated disturbance. We run the causal intervention paradigm on 12 models (6 matched base/instruction-tuned pairs, 8B{--}70B parameters) on GSM8K and MMLU-STEM, adding a content-free filler control matched for length, a re-injection baseline measuring re-prompting and decoding instability, and paired significance testing across conditions. McNemar{'}s test does not detect a significantly larger answer-change rate for removal than for the control in 14 of 24 model{--}task cells (58{\%}) {--} an absence of detected difference, not equivalence {--} the re-prompting/decoding floor alone exceeds 80{\%} answer-change in one model, and a commonly assumed instruction-tuning effect does not replicate across families. We do not conclude that LLM explanations are faithful or unfaithful; we conclude that ACR frequently fails a basic specificity check and cannot alone support causal-faithfulness claims. We release the full pipeline and propose a minimal reporting standard for future studies."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rao-layek-2026-answer">
<titleInfo>
<title>What Does Answer Change Rate Actually Measure? A Specificity Audit of Causal Intervention Tests for Explanation Faithfulness</title>
</titleInfo>
<name type="personal">
<namePart type="given">Durgesh</namePart>
<namePart type="family">Rao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dr.</namePart>
<namePart type="given">Shirshendu</namePart>
<namePart type="family">Layek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>A growing line of work tests whether large language model (LLM) explanations are faithful by intervening on the explanation (removing, perturbing, or swapping it) and checking whether the model’s final answer changes – an Answer Change Rate (ACR), read as causal evidence. We argue this inference requires a control existing work has not consistently reported: whether the answer changes comparably under an unrelated disturbance. We run the causal intervention paradigm on 12 models (6 matched base/instruction-tuned pairs, 8B–70B parameters) on GSM8K and MMLU-STEM, adding a content-free filler control matched for length, a re-injection baseline measuring re-prompting and decoding instability, and paired significance testing across conditions. McNemar’s test does not detect a significantly larger answer-change rate for removal than for the control in 14 of 24 model–task cells (58%) – an absence of detected difference, not equivalence – the re-prompting/decoding floor alone exceeds 80% answer-change in one model, and a commonly assumed instruction-tuning effect does not replicate across families. We do not conclude that LLM explanations are faithful or unfaithful; we conclude that ACR frequently fails a basic specificity check and cannot alone support causal-faithfulness claims. We release the full pipeline and propose a minimal reporting standard for future studies.</abstract>
<identifier type="citekey">rao-layek-2026-answer</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.35/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>441</start>
<end>452</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Does Answer Change Rate Actually Measure? A Specificity Audit of Causal Intervention Tests for Explanation Faithfulness
%A Rao, Durgesh
%A Layek, Dr. Shirshendu
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F rao-layek-2026-answer
%X A growing line of work tests whether large language model (LLM) explanations are faithful by intervening on the explanation (removing, perturbing, or swapping it) and checking whether the model’s final answer changes – an Answer Change Rate (ACR), read as causal evidence. We argue this inference requires a control existing work has not consistently reported: whether the answer changes comparably under an unrelated disturbance. We run the causal intervention paradigm on 12 models (6 matched base/instruction-tuned pairs, 8B–70B parameters) on GSM8K and MMLU-STEM, adding a content-free filler control matched for length, a re-injection baseline measuring re-prompting and decoding instability, and paired significance testing across conditions. McNemar’s test does not detect a significantly larger answer-change rate for removal than for the control in 14 of 24 model–task cells (58%) – an absence of detected difference, not equivalence – the re-prompting/decoding floor alone exceeds 80% answer-change in one model, and a commonly assumed instruction-tuning effect does not replicate across families. We do not conclude that LLM explanations are faithful or unfaithful; we conclude that ACR frequently fails a basic specificity check and cannot alone support causal-faithfulness claims. We release the full pipeline and propose a minimal reporting standard for future studies.
%U https://aclanthology.org/2026.groundlm-1.35/
%P 441-452
Markdown (Informal)
[What Does Answer Change Rate Actually Measure? A Specificity Audit of Causal Intervention Tests for Explanation Faithfulness](https://aclanthology.org/2026.groundlm-1.35/) (Rao & Layek, GroundLM 2026)
ACL