@inproceedings{coumarane-2026-valid,
title = "Valid but Wrong: Domain Invariants and a Cross-Model Plan-Validity Benchmark for Household-Finance {LLM}s",
author = "Coumarane, Vignesh",
editor = "Huang, Jimin and
Lee, Yongjae and
Ananiadou, Sophia and
Chen, Hsin-Hsi and
Takamura, Hiroya and
Chen, Chung-Chi",
booktitle = "Proceedings of the 11th Workshop on Financial Technology and Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.finnlp-1.40/",
pages = "486--490",
ISBN = "979-8-89176-466-8",
abstract = "Today{'}s LLMs are good at solving scoped financial tasks. A plan may be type-correct, schema-valid, and executable, yet not financially sound: double-spending the same dollar, or leaving free employer-match money on the table. We call these valid but wrong, and show that today{'}s evaluations do not consider them. We contribute: (1) a validity hierarchy separating syntactic, schematic, and executional validity from domain validity; (2) a succinct list of domain invariants operationalizing this idea for household finance, divided into conservation and efficiency invariants, grounded in a real six-figure failure in a deployed system; (3) a cross-model measure: we ask eleven current LLMs to plan out a balanced scenario bank (N=585 surface-valid plans) and evaluate each using our invariants. All conform to type, schema, and execution, yet a nonzero fraction break domain invariants which escape detection in surface validity checks. We distinguish a dominance violation over capturable match (unassailable) from an instruction-following violation (does-it-sum) and a contested idle-cash violation. Pooled rates of violating unassailable invariants (the ``floor'') are 15.2{\%} (95{\%} CI 12{--}18{\%}), driven by two small models and by instruction-following; the dominance-only floor is 6.2{\%} pooled and 3.6{\%} for the nine frontier models, whose per-model floors overlap. However, the best model (GPT-5) breaks no unassailable invariants, but parks cash idle on 41{\%} of plans, while a small deployable model fails basic conservation 85{\%} of the time. A median $2,500 in guaranteed employer match is tossed aside per dominance violation. Floors hold even on trap-free controls (7.6\%). Using a tighter definition counting idle cash, which is a preference-based decision, we find an upper bound of 51.3\%. Capability improves on conservation errors but does not close the gap on domain validity.$"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="coumarane-2026-valid">
<titleInfo>
<title>Valid but Wrong: Domain Invariants and a Cross-Model Plan-Validity Benchmark for Household-Finance LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Vignesh</namePart>
<namePart type="family">Coumarane</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Financial Technology and Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jimin</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongjae</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sophia</namePart>
<namePart type="family">Ananiadou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hsin-Hsi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hiroya</namePart>
<namePart type="family">Takamura</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chung-Chi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-466-8</identifier>
</relatedItem>
<abstract>Today’s LLMs are good at solving scoped financial tasks. A plan may be type-correct, schema-valid, and executable, yet not financially sound: double-spending the same dollar, or leaving free employer-match money on the table. We call these valid but wrong, and show that today’s evaluations do not consider them. We contribute: (1) a validity hierarchy separating syntactic, schematic, and executional validity from domain validity; (2) a succinct list of domain invariants operationalizing this idea for household finance, divided into conservation and efficiency invariants, grounded in a real six-figure failure in a deployed system; (3) a cross-model measure: we ask eleven current LLMs to plan out a balanced scenario bank (N=585 surface-valid plans) and evaluate each using our invariants. All conform to type, schema, and execution, yet a nonzero fraction break domain invariants which escape detection in surface validity checks. We distinguish a dominance violation over capturable match (unassailable) from an instruction-following violation (does-it-sum) and a contested idle-cash violation. Pooled rates of violating unassailable invariants (the “floor”) are 15.2% (95% CI 12–18%), driven by two small models and by instruction-following; the dominance-only floor is 6.2% pooled and 3.6% for the nine frontier models, whose per-model floors overlap. However, the best model (GPT-5) breaks no unassailable invariants, but parks cash idle on 41% of plans, while a small deployable model fails basic conservation 85% of the time. A median 2,500 in guaranteed employer match is tossed aside per dominance violation. Floors hold even on trap-free controls (7.6%). Using a tighter definition counting idle cash, which is a preference-based decision, we find an upper bound of 51.3%. Capability improves on conservation errors but does not close the gap on domain validity.</abstract>
<identifier type="citekey">coumarane-2026-valid</identifier>
<location>
<url>https://aclanthology.org/2026.finnlp-1.40/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>486</start>
<end>490</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Valid but Wrong: Domain Invariants and a Cross-Model Plan-Validity Benchmark for Household-Finance LLMs
%A Coumarane, Vignesh
%Y Huang, Jimin
%Y Lee, Yongjae
%Y Ananiadou, Sophia
%Y Chen, Hsin-Hsi
%Y Takamura, Hiroya
%Y Chen, Chung-Chi
%S Proceedings of the 11th Workshop on Financial Technology and Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-466-8
%F coumarane-2026-valid
%X Today’s LLMs are good at solving scoped financial tasks. A plan may be type-correct, schema-valid, and executable, yet not financially sound: double-spending the same dollar, or leaving free employer-match money on the table. We call these valid but wrong, and show that today’s evaluations do not consider them. We contribute: (1) a validity hierarchy separating syntactic, schematic, and executional validity from domain validity; (2) a succinct list of domain invariants operationalizing this idea for household finance, divided into conservation and efficiency invariants, grounded in a real six-figure failure in a deployed system; (3) a cross-model measure: we ask eleven current LLMs to plan out a balanced scenario bank (N=585 surface-valid plans) and evaluate each using our invariants. All conform to type, schema, and execution, yet a nonzero fraction break domain invariants which escape detection in surface validity checks. We distinguish a dominance violation over capturable match (unassailable) from an instruction-following violation (does-it-sum) and a contested idle-cash violation. Pooled rates of violating unassailable invariants (the “floor”) are 15.2% (95% CI 12–18%), driven by two small models and by instruction-following; the dominance-only floor is 6.2% pooled and 3.6% for the nine frontier models, whose per-model floors overlap. However, the best model (GPT-5) breaks no unassailable invariants, but parks cash idle on 41% of plans, while a small deployable model fails basic conservation 85% of the time. A median 2,500 in guaranteed employer match is tossed aside per dominance violation. Floors hold even on trap-free controls (7.6%). Using a tighter definition counting idle cash, which is a preference-based decision, we find an upper bound of 51.3%. Capability improves on conservation errors but does not close the gap on domain validity.
%U https://aclanthology.org/2026.finnlp-1.40/
%P 486-490
Markdown (Informal)
[Valid but Wrong: Domain Invariants and a Cross-Model Plan-Validity Benchmark for Household-Finance LLMs](https://aclanthology.org/2026.finnlp-1.40/) (Coumarane, FinNLP 2026)
ACL