@inproceedings{thokal-2026-deterministic,
title = "What Deterministic Verification Actually Fixes in Financial {QA}: A Diagnostic Analysis of Rule Heterogeneity Across Datasets",
author = "Thokal, Aaditya Anand",
editor = "Huang, Jimin and
Lee, Yongjae and
Ananiadou, Sophia and
Chen, Hsin-Hsi and
Takamura, Hiroya and
Chen, Chung-Chi",
booktitle = "Proceedings of the 11th Workshop on Financial Technology and Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.finnlp-1.16/",
pages = "193--200",
ISBN = "979-8-89176-466-8",
abstract = "Deterministic, parameter-free post-hoc correction rules are an attractive way to address numerical errors in financial question answering: they are simple, cheap, and fully auditable. We ask whether such rules actually behave like general-purpose correctors, or whether their effect is contingent on properties of the dataset and model output distribution they are applied to. We evaluate a two-rule deterministic verification layer (DVL) {---} scale{\_}mul100 and scale{\_}div100 {---} on FinQA (n=873) and TAT-QA (n=718) using Mistral-7B-Instruct-v0.3. This same, fixed model-plus-rule configuration produces a striking cross-dataset divergence: harmful on FinQA (37.34{\%}{\textrightarrow}16.04{\%}, {\ensuremath{-}}21.31pp, McNemar p=1.76{\texttimes}10{\ensuremath{^-}}{\textthreesuperior}{\ensuremath{^9}}) and beneficial on TAT-QA (26.32{\%}{\textrightarrow}30.64{\%}, +4.32pp, p=5.15{\texttimes}10{\ensuremath{^-}}{\textthreesuperior}). A rule-level transition analysis shows scale{\_}mul100 is highly precise on TAT-QA (87.95{\%}) but overwhelmingly imprecise on FinQA (3.39{\%}), while scale{\_}div100 produces zero observed corrections and only harm in both settings; removing it raises TAT-QA accuracy further, to 35.10{\%}. We do not find evidence that these rules correct arithmetic, retrieval, or semantic reasoning failures. We conclude that this specific pair of scaling rules is better understood as a targeted, dataset-contingent representation normalizer than as a general reasoning corrector, and we characterize the boundary conditions under which it helps or hurts."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="thokal-2026-deterministic">
<titleInfo>
<title>What Deterministic Verification Actually Fixes in Financial QA: A Diagnostic Analysis of Rule Heterogeneity Across Datasets</title>
</titleInfo>
<name type="personal">
<namePart type="given">Aaditya</namePart>
<namePart type="given">Anand</namePart>
<namePart type="family">Thokal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 11th Workshop on Financial Technology and Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jimin</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongjae</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sophia</namePart>
<namePart type="family">Ananiadou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hsin-Hsi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hiroya</namePart>
<namePart type="family">Takamura</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chung-Chi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-466-8</identifier>
</relatedItem>
<abstract>Deterministic, parameter-free post-hoc correction rules are an attractive way to address numerical errors in financial question answering: they are simple, cheap, and fully auditable. We ask whether such rules actually behave like general-purpose correctors, or whether their effect is contingent on properties of the dataset and model output distribution they are applied to. We evaluate a two-rule deterministic verification layer (DVL) — scale_mul100 and scale_div100 — on FinQA (n=873) and TAT-QA (n=718) using Mistral-7B-Instruct-v0.3. This same, fixed model-plus-rule configuration produces a striking cross-dataset divergence: harmful on FinQA (37.34%→16.04%, \ensuremath-21.31pp, McNemar p=1.76×10\ensuremath⁻³\ensuremath⁹) and beneficial on TAT-QA (26.32%→30.64%, +4.32pp, p=5.15×10\ensuremath⁻³). A rule-level transition analysis shows scale_mul100 is highly precise on TAT-QA (87.95%) but overwhelmingly imprecise on FinQA (3.39%), while scale_div100 produces zero observed corrections and only harm in both settings; removing it raises TAT-QA accuracy further, to 35.10%. We do not find evidence that these rules correct arithmetic, retrieval, or semantic reasoning failures. We conclude that this specific pair of scaling rules is better understood as a targeted, dataset-contingent representation normalizer than as a general reasoning corrector, and we characterize the boundary conditions under which it helps or hurts.</abstract>
<identifier type="citekey">thokal-2026-deterministic</identifier>
<location>
<url>https://aclanthology.org/2026.finnlp-1.16/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>193</start>
<end>200</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Deterministic Verification Actually Fixes in Financial QA: A Diagnostic Analysis of Rule Heterogeneity Across Datasets
%A Thokal, Aaditya Anand
%Y Huang, Jimin
%Y Lee, Yongjae
%Y Ananiadou, Sophia
%Y Chen, Hsin-Hsi
%Y Takamura, Hiroya
%Y Chen, Chung-Chi
%S Proceedings of the 11th Workshop on Financial Technology and Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-466-8
%F thokal-2026-deterministic
%X Deterministic, parameter-free post-hoc correction rules are an attractive way to address numerical errors in financial question answering: they are simple, cheap, and fully auditable. We ask whether such rules actually behave like general-purpose correctors, or whether their effect is contingent on properties of the dataset and model output distribution they are applied to. We evaluate a two-rule deterministic verification layer (DVL) — scale_mul100 and scale_div100 — on FinQA (n=873) and TAT-QA (n=718) using Mistral-7B-Instruct-v0.3. This same, fixed model-plus-rule configuration produces a striking cross-dataset divergence: harmful on FinQA (37.34%→16.04%, \ensuremath-21.31pp, McNemar p=1.76×10\ensuremath⁻³\ensuremath⁹) and beneficial on TAT-QA (26.32%→30.64%, +4.32pp, p=5.15×10\ensuremath⁻³). A rule-level transition analysis shows scale_mul100 is highly precise on TAT-QA (87.95%) but overwhelmingly imprecise on FinQA (3.39%), while scale_div100 produces zero observed corrections and only harm in both settings; removing it raises TAT-QA accuracy further, to 35.10%. We do not find evidence that these rules correct arithmetic, retrieval, or semantic reasoning failures. We conclude that this specific pair of scaling rules is better understood as a targeted, dataset-contingent representation normalizer than as a general reasoning corrector, and we characterize the boundary conditions under which it helps or hurts.
%U https://aclanthology.org/2026.finnlp-1.16/
%P 193-200
Markdown (Informal)
[What Deterministic Verification Actually Fixes in Financial QA: A Diagnostic Analysis of Rule Heterogeneity Across Datasets](https://aclanthology.org/2026.finnlp-1.16/) (Thokal, FinNLP 2026)
ACL