@inproceedings{buzato-etal-2026-role,
title = "On the Role of Supporting Evidence in {LLM} Reasoning: A Causal Analysis of Legal {QA} with Activation Patching",
author = "Buzato, Dalmo and
Franco, Fernanda Sathler Rocha and
Caldeira, Caio Alves and
Santos, Isabela Carolina Ribeiro and
Cunha, Evandro L. T. P. and
Calais, Pedro and
Meira Jr., Wagner and
Lacerda, Anisio",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1562/",
pages = "35623--35635",
ISBN = "979-8-89176-459-0",
abstract = "As large language models (LLMs) are increasingly considered for high-stakes legal applications, a central question for trust and auditability is whether their outputs reflect genuine reasoning over the supplied legal materials, or merely surface statistical patterns absorbed during pre-training. We address this question at the mechanistic level by applying activation patching, a causal-mediation technique, to four open-source multilingual LLMs on two bar-exam datasets: U.S. MBE and Brazilian OAB. For each multiple-choice question we contrast a gold supporting passage with an irrelevant-but-plausible counterfactual. We then surgically replace residual-stream activations across layers and token regions and measure recovery of the logit difference. Gold passages exert a measurable but model- and linguistic-dependent causal effect; integration concentrates in the upper layers (a late-binding signature); and the size of the effect is strongly mediated by the linguistic and discourse structure of the passage itself, not just its relevance. We introduce a four-cluster taxonomy that operationalises these regimes and identifies cases where retrieved evidence is causally decisive, inert, or actively harmful."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="buzato-etal-2026-role">
<titleInfo>
<title>On the Role of Supporting Evidence in LLM Reasoning: A Causal Analysis of Legal QA with Activation Patching</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dalmo</namePart>
<namePart type="family">Buzato</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fernanda</namePart>
<namePart type="given">Sathler</namePart>
<namePart type="given">Rocha</namePart>
<namePart type="family">Franco</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Caio</namePart>
<namePart type="given">Alves</namePart>
<namePart type="family">Caldeira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Isabela</namePart>
<namePart type="given">Carolina</namePart>
<namePart type="given">Ribeiro</namePart>
<namePart type="family">Santos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Evandro</namePart>
<namePart type="given">L</namePart>
<namePart type="given">T</namePart>
<namePart type="given">P</namePart>
<namePart type="family">Cunha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pedro</namePart>
<namePart type="family">Calais</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wagner</namePart>
<namePart type="family">Meira Jr.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anisio</namePart>
<namePart type="family">Lacerda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>As large language models (LLMs) are increasingly considered for high-stakes legal applications, a central question for trust and auditability is whether their outputs reflect genuine reasoning over the supplied legal materials, or merely surface statistical patterns absorbed during pre-training. We address this question at the mechanistic level by applying activation patching, a causal-mediation technique, to four open-source multilingual LLMs on two bar-exam datasets: U.S. MBE and Brazilian OAB. For each multiple-choice question we contrast a gold supporting passage with an irrelevant-but-plausible counterfactual. We then surgically replace residual-stream activations across layers and token regions and measure recovery of the logit difference. Gold passages exert a measurable but model- and linguistic-dependent causal effect; integration concentrates in the upper layers (a late-binding signature); and the size of the effect is strongly mediated by the linguistic and discourse structure of the passage itself, not just its relevance. We introduce a four-cluster taxonomy that operationalises these regimes and identifies cases where retrieved evidence is causally decisive, inert, or actively harmful.</abstract>
<identifier type="citekey">buzato-etal-2026-role</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1562/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>35623</start>
<end>35635</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T On the Role of Supporting Evidence in LLM Reasoning: A Causal Analysis of Legal QA with Activation Patching
%A Buzato, Dalmo
%A Franco, Fernanda Sathler Rocha
%A Caldeira, Caio Alves
%A Santos, Isabela Carolina Ribeiro
%A Cunha, Evandro L. T. P.
%A Calais, Pedro
%A Meira Jr., Wagner
%A Lacerda, Anisio
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F buzato-etal-2026-role
%X As large language models (LLMs) are increasingly considered for high-stakes legal applications, a central question for trust and auditability is whether their outputs reflect genuine reasoning over the supplied legal materials, or merely surface statistical patterns absorbed during pre-training. We address this question at the mechanistic level by applying activation patching, a causal-mediation technique, to four open-source multilingual LLMs on two bar-exam datasets: U.S. MBE and Brazilian OAB. For each multiple-choice question we contrast a gold supporting passage with an irrelevant-but-plausible counterfactual. We then surgically replace residual-stream activations across layers and token regions and measure recovery of the logit difference. Gold passages exert a measurable but model- and linguistic-dependent causal effect; integration concentrates in the upper layers (a late-binding signature); and the size of the effect is strongly mediated by the linguistic and discourse structure of the passage itself, not just its relevance. We introduce a four-cluster taxonomy that operationalises these regimes and identifies cases where retrieved evidence is causally decisive, inert, or actively harmful.
%U https://aclanthology.org/2026.emnlp-main.1562/
%P 35623-35635
Markdown (Informal)
[On the Role of Supporting Evidence in LLM Reasoning: A Causal Analysis of Legal QA with Activation Patching](https://aclanthology.org/2026.emnlp-main.1562/) (Buzato et al., EMNLP 2026)
ACL
- Dalmo Buzato, Fernanda Sathler Rocha Franco, Caio Alves Caldeira, Isabela Carolina Ribeiro Santos, Evandro L. T. P. Cunha, Pedro Calais, Wagner Meira Jr., and Anisio Lacerda. 2026. On the Role of Supporting Evidence in LLM Reasoning: A Causal Analysis of Legal QA with Activation Patching. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 35623–35635, Budapest, Hungary. Association for Computational Linguistics.