@inproceedings{harish-2026-cost,
title = "The Cost of Incoherent Context Grows with Capability: A Control for Exemplar Retrieval",
author = "Harish, Srinivas",
editor = "Valentino, Marco and
Ferreira, Deborah and
Thayaparan, Mokanarangan and
Ranaldi, Leonardo and
Cheng, Jo-Ku and
Freitas, Andre",
booktitle = "The Proceedings of the 4th Workshop on Mathematical Natural Language Processing ({M}ath{NLP} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.mathnlp-main.7/",
pages = "86--100",
ISBN = "979-8-89176-468-2",
abstract = "Comparisons of retrieved and randomly selected exemplars in mathematical in-context learning confound query relevance with changes in the demonstration pool. We separate these effects by assigning each query the demonstrations retrieved for another query. This derangement preserves the aggregate demonstrations, counts, token budget and retrieval-induced selection bias while changing the query-to-exemplar assignment.\par On GSM8K, derangement changes accuracy by {\ensuremath{-}}0.24 pp (90{\%} CI [{\ensuremath{-}}0.85, +0.37]). The pooled effect is equivalent to zero within 1.0 pp across conditions that include embedding retrieval, a gold-computational-graph oracle and a 7B model. On MathQA, the change is {\ensuremath{-}}8.36 pp. Near-duplicates occur above cosine 0.95 for 64.6{\%} of MathQA queries and no GSM8K queries. Removing them changes the matching term by {\ensuremath{-}}13.56 pp relative to queries unaffected by removal.\par Pairing each retrieved problem with another retrieved rationale changes accuracy by {\ensuremath{-}}22.40 to {\ensuremath{-}}52.24 pp. Across 8 conditions and 3 model families, this change correlates with excess copying of demonstration answers (Spearman {\ensuremath{-}}0.976). Across 7 models, the correlation between the accuracy change and unpermuted accuracy is +0.286. These results support reporting pool near-duplicate density alongside retrieval gains and checking problem-rationale alignment during prompt construction, since retrieval metrics remain unchanged under this within-demonstration permutation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="harish-2026-cost">
<titleInfo>
<title>The Cost of Incoherent Context Grows with Capability: A Control for Exemplar Retrieval</title>
</titleInfo>
<name type="personal">
<namePart type="given">Srinivas</namePart>
<namePart type="family">Harish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the 4th Workshop on Mathematical Natural Language Processing (MathNLP 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Valentino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Deborah</namePart>
<namePart type="family">Ferreira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mokanarangan</namePart>
<namePart type="family">Thayaparan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leonardo</namePart>
<namePart type="family">Ranaldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jo-Ku</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andre</namePart>
<namePart type="family">Freitas</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-468-2</identifier>
</relatedItem>
<abstract>Comparisons of retrieved and randomly selected exemplars in mathematical in-context learning confound query relevance with changes in the demonstration pool. We separate these effects by assigning each query the demonstrations retrieved for another query. This derangement preserves the aggregate demonstrations, counts, token budget and retrieval-induced selection bias while changing the query-to-exemplar assignment.\par On GSM8K, derangement changes accuracy by \ensuremath-0.24 pp (90% CI [\ensuremath-0.85, +0.37]). The pooled effect is equivalent to zero within 1.0 pp across conditions that include embedding retrieval, a gold-computational-graph oracle and a 7B model. On MathQA, the change is \ensuremath-8.36 pp. Near-duplicates occur above cosine 0.95 for 64.6% of MathQA queries and no GSM8K queries. Removing them changes the matching term by \ensuremath-13.56 pp relative to queries unaffected by removal.\par Pairing each retrieved problem with another retrieved rationale changes accuracy by \ensuremath-22.40 to \ensuremath-52.24 pp. Across 8 conditions and 3 model families, this change correlates with excess copying of demonstration answers (Spearman \ensuremath-0.976). Across 7 models, the correlation between the accuracy change and unpermuted accuracy is +0.286. These results support reporting pool near-duplicate density alongside retrieval gains and checking problem-rationale alignment during prompt construction, since retrieval metrics remain unchanged under this within-demonstration permutation.</abstract>
<identifier type="citekey">harish-2026-cost</identifier>
<location>
<url>https://aclanthology.org/2026.mathnlp-main.7/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>86</start>
<end>100</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Cost of Incoherent Context Grows with Capability: A Control for Exemplar Retrieval
%A Harish, Srinivas
%Y Valentino, Marco
%Y Ferreira, Deborah
%Y Thayaparan, Mokanarangan
%Y Ranaldi, Leonardo
%Y Cheng, Jo-Ku
%Y Freitas, Andre
%S The Proceedings of the 4th Workshop on Mathematical Natural Language Processing (MathNLP 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-468-2
%F harish-2026-cost
%X Comparisons of retrieved and randomly selected exemplars in mathematical in-context learning confound query relevance with changes in the demonstration pool. We separate these effects by assigning each query the demonstrations retrieved for another query. This derangement preserves the aggregate demonstrations, counts, token budget and retrieval-induced selection bias while changing the query-to-exemplar assignment.\par On GSM8K, derangement changes accuracy by \ensuremath-0.24 pp (90% CI [\ensuremath-0.85, +0.37]). The pooled effect is equivalent to zero within 1.0 pp across conditions that include embedding retrieval, a gold-computational-graph oracle and a 7B model. On MathQA, the change is \ensuremath-8.36 pp. Near-duplicates occur above cosine 0.95 for 64.6% of MathQA queries and no GSM8K queries. Removing them changes the matching term by \ensuremath-13.56 pp relative to queries unaffected by removal.\par Pairing each retrieved problem with another retrieved rationale changes accuracy by \ensuremath-22.40 to \ensuremath-52.24 pp. Across 8 conditions and 3 model families, this change correlates with excess copying of demonstration answers (Spearman \ensuremath-0.976). Across 7 models, the correlation between the accuracy change and unpermuted accuracy is +0.286. These results support reporting pool near-duplicate density alongside retrieval gains and checking problem-rationale alignment during prompt construction, since retrieval metrics remain unchanged under this within-demonstration permutation.
%U https://aclanthology.org/2026.mathnlp-main.7/
%P 86-100
Markdown (Informal)
[The Cost of Incoherent Context Grows with Capability: A Control for Exemplar Retrieval](https://aclanthology.org/2026.mathnlp-main.7/) (Harish, MathNLP 2026)
ACL