@inproceedings{sayer-mathur-2026-terms,
title = "Terms and Conditions Apply: Evaluating Consumer {LLM} Reliability in {NDA} Interpretation to Investigate the Non-Linear Role of Reasoning",
author = "Sayer, Simeon and
Mathur, Shaurya Jain",
editor = "Aletras, Nikolaos and
Chalkidis, Ilias and
Barrett, Leslie and
Goanț{\u{a}}, C{\u{a}}t{\u{a}}lina and
Preoțiuc-Pietro, Daniel and
Spanakis, Gerasimos and
Xu, Shanshan",
booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nllp-1.20/",
pages = "237--256",
ISBN = "979-8-89176-494-1",
abstract = "This paper uses the standardized structure of NDA{'}s to first explore how advances in reasoning models impact everyday legal tasks for the average consumer, and second to use the comparative model performance to examine pitfalls of traditional model benchmarking. Typically, Large language model (LLM) benchmarks express performance as a single score. This makes progress easy to compare, but not necessarily easy to understand: a higher-scoring model may gain one capability while losing another that the benchmark average no longer makes visible. This paper uses consumer interpretation of non-disclosure agreements as a practical, standardised case study, while simultaneously highlighting the relative reliability of an everyday legal task. Gemini 2.5 Flash-Lite and GPT-5.5 evaluated 2,091 NDA-hypothesis pairs derived from a human classified as either Entailment, Contradiction, or NotMentioned. While GPT performed better overall, particularly in recognising contradictions, Gemini remained uniquely correct in 247 cases, particularly in asserting that claims were NotMentioned. GPT inferred unfounded hypotheses more readily and sometimes appeared to `overthink' and reason itself into false Contradictions. The lack of correlation between NDA performance and reasoning means that while GPT alone was correct 80.2{\%} of the time, a hypothetical model selector that optimally chose the correct system could achieve 92.0{\%} accuracy. The results show both that advances in model capabilities translate into improved legal advice for a typical consumer and that a rising benchmark score can conceal an exchange of capabilities rather than a uniform improvement. The results also point to broader lessons for similar benchmarking so that evaluations report error direction, correctness overlap, and task-level reversals alongside aggregate accuracy. For consumers, the results indicate that while LLMs remain useful for preliminary contract review, the results do not support treating them as autonomous legal advisers."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sayer-mathur-2026-terms">
<titleInfo>
<title>Terms and Conditions Apply: Evaluating Consumer LLM Reliability in NDA Interpretation to Investigate the Non-Linear Role of Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Simeon</namePart>
<namePart type="family">Sayer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaurya</namePart>
<namePart type="given">Jain</namePart>
<namePart type="family">Mathur</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Natural Legal Language Processing Workshop 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Chalkidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leslie</namePart>
<namePart type="family">Barrett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cătălina</namePart>
<namePart type="family">Goanță</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoțiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Spanakis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-494-1</identifier>
</relatedItem>
<abstract>This paper uses the standardized structure of NDA’s to first explore how advances in reasoning models impact everyday legal tasks for the average consumer, and second to use the comparative model performance to examine pitfalls of traditional model benchmarking. Typically, Large language model (LLM) benchmarks express performance as a single score. This makes progress easy to compare, but not necessarily easy to understand: a higher-scoring model may gain one capability while losing another that the benchmark average no longer makes visible. This paper uses consumer interpretation of non-disclosure agreements as a practical, standardised case study, while simultaneously highlighting the relative reliability of an everyday legal task. Gemini 2.5 Flash-Lite and GPT-5.5 evaluated 2,091 NDA-hypothesis pairs derived from a human classified as either Entailment, Contradiction, or NotMentioned. While GPT performed better overall, particularly in recognising contradictions, Gemini remained uniquely correct in 247 cases, particularly in asserting that claims were NotMentioned. GPT inferred unfounded hypotheses more readily and sometimes appeared to ‘overthink’ and reason itself into false Contradictions. The lack of correlation between NDA performance and reasoning means that while GPT alone was correct 80.2% of the time, a hypothetical model selector that optimally chose the correct system could achieve 92.0% accuracy. The results show both that advances in model capabilities translate into improved legal advice for a typical consumer and that a rising benchmark score can conceal an exchange of capabilities rather than a uniform improvement. The results also point to broader lessons for similar benchmarking so that evaluations report error direction, correctness overlap, and task-level reversals alongside aggregate accuracy. For consumers, the results indicate that while LLMs remain useful for preliminary contract review, the results do not support treating them as autonomous legal advisers.</abstract>
<identifier type="citekey">sayer-mathur-2026-terms</identifier>
<location>
<url>https://aclanthology.org/2026.nllp-1.20/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>237</start>
<end>256</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Terms and Conditions Apply: Evaluating Consumer LLM Reliability in NDA Interpretation to Investigate the Non-Linear Role of Reasoning
%A Sayer, Simeon
%A Mathur, Shaurya Jain
%Y Aletras, Nikolaos
%Y Chalkidis, Ilias
%Y Barrett, Leslie
%Y Goanță, Cătălina
%Y Preoțiuc-Pietro, Daniel
%Y Spanakis, Gerasimos
%Y Xu, Shanshan
%S Proceedings of the Natural Legal Language Processing Workshop 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-494-1
%F sayer-mathur-2026-terms
%X This paper uses the standardized structure of NDA’s to first explore how advances in reasoning models impact everyday legal tasks for the average consumer, and second to use the comparative model performance to examine pitfalls of traditional model benchmarking. Typically, Large language model (LLM) benchmarks express performance as a single score. This makes progress easy to compare, but not necessarily easy to understand: a higher-scoring model may gain one capability while losing another that the benchmark average no longer makes visible. This paper uses consumer interpretation of non-disclosure agreements as a practical, standardised case study, while simultaneously highlighting the relative reliability of an everyday legal task. Gemini 2.5 Flash-Lite and GPT-5.5 evaluated 2,091 NDA-hypothesis pairs derived from a human classified as either Entailment, Contradiction, or NotMentioned. While GPT performed better overall, particularly in recognising contradictions, Gemini remained uniquely correct in 247 cases, particularly in asserting that claims were NotMentioned. GPT inferred unfounded hypotheses more readily and sometimes appeared to ‘overthink’ and reason itself into false Contradictions. The lack of correlation between NDA performance and reasoning means that while GPT alone was correct 80.2% of the time, a hypothetical model selector that optimally chose the correct system could achieve 92.0% accuracy. The results show both that advances in model capabilities translate into improved legal advice for a typical consumer and that a rising benchmark score can conceal an exchange of capabilities rather than a uniform improvement. The results also point to broader lessons for similar benchmarking so that evaluations report error direction, correctness overlap, and task-level reversals alongside aggregate accuracy. For consumers, the results indicate that while LLMs remain useful for preliminary contract review, the results do not support treating them as autonomous legal advisers.
%U https://aclanthology.org/2026.nllp-1.20/
%P 237-256
Markdown (Informal)
[Terms and Conditions Apply: Evaluating Consumer LLM Reliability in NDA Interpretation to Investigate the Non-Linear Role of Reasoning](https://aclanthology.org/2026.nllp-1.20/) (Sayer & Mathur, NLLP 2026)
ACL