@inproceedings{kale-2026-beyond,
title = "Beyond Calibration: Temporal {LLM} Confidence Assessment Across Languages and Local Settings",
author = "Kale, Sahil",
editor = "Zhang, Chen and
Allaway, Emily and
M{'}hamdi, Meryem and
Limkonchotiwat, Peerat and
Bai, Richard He and
T.Y.S.S., Santosh and
Thapa, Surendrabikram and
Li, Yinqiao",
booktitle = "Proceedings of the 10th Widening {NLP} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.winlp-main.2/",
pages = "10--19",
ISBN = "979-8-89176-472-9",
abstract = "Large language models are increasingly expected to estimate their own uncertainty, with confidence signals being used to guide retrieval, adaptive compute allocation, routing, and selective abstention. In this work, we investigate whether confidence signals measured at different temporal stages of problem solving encode distinct forms of epistemic information, and whether such signals remain reliable beyond English-centric settings. Inspired by classical meta-memory theory, we distinguish between anticipatory pre-solve confidence (Feeling-of-Knowing, FOK) and retrospective post-solve confidence (Judgement-of-Learning, JOL). We evaluate these signals across frontier LLMs, multiple reasoning domains, translated Marathi benchmarks, and a newly constructed benchmark of genuinely local Marathi factual questions. Across settings, the cheaper pre-solve signal remains weakly calibrated, while post-solve confidence generally provides stronger epistemic alignment despite additional inference cost. We further find that confidence behaviour transfers surprisingly well under direct language translation, with calibration trends largely preserved between English and Marathi. In contrast, confidence reliability degrades substantially on local Marathi knowledge, suggesting that linguistic transfer alone is insufficient to capture the challenges posed by region-specific information needs. Finally, hallucinated responses maintain persistently elevated confidence even after answer generation, highlighting important limitations of current LLM self-assessment mechanisms."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kale-2026-beyond">
<titleInfo>
<title>Beyond Calibration: Temporal LLM Confidence Assessment Across Languages and Local Settings</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sahil</namePart>
<namePart type="family">Kale</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 10th Widening NLP Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Emily</namePart>
<namePart type="family">Allaway</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Meryem</namePart>
<namePart type="family">M’hamdi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peerat</namePart>
<namePart type="family">Limkonchotiwat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Richard</namePart>
<namePart type="given">He</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Surendrabikram</namePart>
<namePart type="family">Thapa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinqiao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-472-9</identifier>
</relatedItem>
<abstract>Large language models are increasingly expected to estimate their own uncertainty, with confidence signals being used to guide retrieval, adaptive compute allocation, routing, and selective abstention. In this work, we investigate whether confidence signals measured at different temporal stages of problem solving encode distinct forms of epistemic information, and whether such signals remain reliable beyond English-centric settings. Inspired by classical meta-memory theory, we distinguish between anticipatory pre-solve confidence (Feeling-of-Knowing, FOK) and retrospective post-solve confidence (Judgement-of-Learning, JOL). We evaluate these signals across frontier LLMs, multiple reasoning domains, translated Marathi benchmarks, and a newly constructed benchmark of genuinely local Marathi factual questions. Across settings, the cheaper pre-solve signal remains weakly calibrated, while post-solve confidence generally provides stronger epistemic alignment despite additional inference cost. We further find that confidence behaviour transfers surprisingly well under direct language translation, with calibration trends largely preserved between English and Marathi. In contrast, confidence reliability degrades substantially on local Marathi knowledge, suggesting that linguistic transfer alone is insufficient to capture the challenges posed by region-specific information needs. Finally, hallucinated responses maintain persistently elevated confidence even after answer generation, highlighting important limitations of current LLM self-assessment mechanisms.</abstract>
<identifier type="citekey">kale-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.winlp-main.2/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>10</start>
<end>19</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Calibration: Temporal LLM Confidence Assessment Across Languages and Local Settings
%A Kale, Sahil
%Y Zhang, Chen
%Y Allaway, Emily
%Y M’hamdi, Meryem
%Y Limkonchotiwat, Peerat
%Y Bai, Richard He
%Y T.Y.S.S., Santosh
%Y Thapa, Surendrabikram
%Y Li, Yinqiao
%S Proceedings of the 10th Widening NLP Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-472-9
%F kale-2026-beyond
%X Large language models are increasingly expected to estimate their own uncertainty, with confidence signals being used to guide retrieval, adaptive compute allocation, routing, and selective abstention. In this work, we investigate whether confidence signals measured at different temporal stages of problem solving encode distinct forms of epistemic information, and whether such signals remain reliable beyond English-centric settings. Inspired by classical meta-memory theory, we distinguish between anticipatory pre-solve confidence (Feeling-of-Knowing, FOK) and retrospective post-solve confidence (Judgement-of-Learning, JOL). We evaluate these signals across frontier LLMs, multiple reasoning domains, translated Marathi benchmarks, and a newly constructed benchmark of genuinely local Marathi factual questions. Across settings, the cheaper pre-solve signal remains weakly calibrated, while post-solve confidence generally provides stronger epistemic alignment despite additional inference cost. We further find that confidence behaviour transfers surprisingly well under direct language translation, with calibration trends largely preserved between English and Marathi. In contrast, confidence reliability degrades substantially on local Marathi knowledge, suggesting that linguistic transfer alone is insufficient to capture the challenges posed by region-specific information needs. Finally, hallucinated responses maintain persistently elevated confidence even after answer generation, highlighting important limitations of current LLM self-assessment mechanisms.
%U https://aclanthology.org/2026.winlp-main.2/
%P 10-19
Markdown (Informal)
[Beyond Calibration: Temporal LLM Confidence Assessment Across Languages and Local Settings](https://aclanthology.org/2026.winlp-main.2/) (Kale, WiNLP 2026)
ACL