@inproceedings{thlija-etal-2026-drives,
title = "What Drives an {LLM}{'}s Legal Reasoning? Measuring Faithfulness at the Concept Level",
author = "Thlija, Yassine and
Held, Lena and
Habernal, Ivan",
editor = "Aletras, Nikolaos and
Chalkidis, Ilias and
Barrett, Leslie and
Goanț{\u{a}}, C{\u{a}}t{\u{a}}lina and
Preoțiuc-Pietro, Daniel and
Spanakis, Gerasimos and
Xu, Shanshan",
booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nllp-1.26/",
pages = "303--320",
ISBN = "979-8-89176-494-1",
abstract = "Faithfulness assesses whether a model{'}s explanation reflects what actually drove its answer, which is different from assessing answer correctness. But does faithfulness also apply in expert domains such as law, where the relevant factors that decide the answer, also called concepts, are diffuse and only implicitly expressed? We measure faithfulness in two expert-annotated legal reasoning multiple-choice question answering (QA) datasets and show that existing methods are not tailor-made for the legal domain. We therefore introduce a detailed, per-concept measure which reveals how each concept is being relied on and expressed by a model. We test our method with a small, open-weight LLM and find that faithfulness is generally low on legal datasets and that simple supervised fine-tuning lowers faithfulness even further, despite giving plausible legal explanations."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="thlija-etal-2026-drives">
<titleInfo>
<title>What Drives an LLM’s Legal Reasoning? Measuring Faithfulness at the Concept Level</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yassine</namePart>
<namePart type="family">Thlija</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lena</namePart>
<namePart type="family">Held</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ivan</namePart>
<namePart type="family">Habernal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Natural Legal Language Processing Workshop 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Chalkidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leslie</namePart>
<namePart type="family">Barrett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cătălina</namePart>
<namePart type="family">Goanță</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoțiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Spanakis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-494-1</identifier>
</relatedItem>
<abstract>Faithfulness assesses whether a model’s explanation reflects what actually drove its answer, which is different from assessing answer correctness. But does faithfulness also apply in expert domains such as law, where the relevant factors that decide the answer, also called concepts, are diffuse and only implicitly expressed? We measure faithfulness in two expert-annotated legal reasoning multiple-choice question answering (QA) datasets and show that existing methods are not tailor-made for the legal domain. We therefore introduce a detailed, per-concept measure which reveals how each concept is being relied on and expressed by a model. We test our method with a small, open-weight LLM and find that faithfulness is generally low on legal datasets and that simple supervised fine-tuning lowers faithfulness even further, despite giving plausible legal explanations.</abstract>
<identifier type="citekey">thlija-etal-2026-drives</identifier>
<location>
<url>https://aclanthology.org/2026.nllp-1.26/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>303</start>
<end>320</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Drives an LLM’s Legal Reasoning? Measuring Faithfulness at the Concept Level
%A Thlija, Yassine
%A Held, Lena
%A Habernal, Ivan
%Y Aletras, Nikolaos
%Y Chalkidis, Ilias
%Y Barrett, Leslie
%Y Goanță, Cătălina
%Y Preoțiuc-Pietro, Daniel
%Y Spanakis, Gerasimos
%Y Xu, Shanshan
%S Proceedings of the Natural Legal Language Processing Workshop 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-494-1
%F thlija-etal-2026-drives
%X Faithfulness assesses whether a model’s explanation reflects what actually drove its answer, which is different from assessing answer correctness. But does faithfulness also apply in expert domains such as law, where the relevant factors that decide the answer, also called concepts, are diffuse and only implicitly expressed? We measure faithfulness in two expert-annotated legal reasoning multiple-choice question answering (QA) datasets and show that existing methods are not tailor-made for the legal domain. We therefore introduce a detailed, per-concept measure which reveals how each concept is being relied on and expressed by a model. We test our method with a small, open-weight LLM and find that faithfulness is generally low on legal datasets and that simple supervised fine-tuning lowers faithfulness even further, despite giving plausible legal explanations.
%U https://aclanthology.org/2026.nllp-1.26/
%P 303-320
Markdown (Informal)
[What Drives an LLM’s Legal Reasoning? Measuring Faithfulness at the Concept Level](https://aclanthology.org/2026.nllp-1.26/) (Thlija et al., NLLP 2026)
ACL