@inproceedings{pit-vylomova-2026-context,
title = "When Context Sensitivity Isn{'}t Enough: An Argument Against Likelihood-Based Probes for Evaluating {LLM}-Simulated Students",
author = "Pit, Pagnarith and
Vylomova, Ekaterina",
editor = "Xu, Shanshan and
T.Y.S.S., Santosh and
Kurniawan, Kemal and
Peng, Siyao and
Wallbridge, Sarenne and
Koto, Fajri and
Dziri, Nouha and
Plank, Barbara",
booktitle = "Proceedings of the First Workshop on Pluralistic {AI} {NLP}: Diversity-aware, Sociotechnical, Responsible Alignment ({PANDORA} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.pandora-1.7/",
pages = "135--148",
ISBN = "979-8-89176-484-2",
abstract = "Evaluating language models as student simulators currently focuses on whether their dialogue looks realistic, yet it is unclear whether these models actually respond to the quality of teaching the way human students do. However, this has not deterred prior works from using a model{'}s uncertainty, measured as the log-likelihood of a fixed correct answer, as a proxy for student behaviour. This approach has been used to simulate student responses, estimate question difficulty, and study ``learning''. In this work, we test whether this proxy can detect sensitivity to instructional quality. We evaluate five language models on expert-rated tutoring dialogues under tightly controlled conditions. All models reliably distinguish relevant tutoring context from irrelevant context. However, this contextual sensitivity does not extend to pedagogical sensitivity: higher-quality instruction almost never increases the likelihood of the correct answer, and reasoning-oriented models sometimes show the opposite pattern. We argue that this failure is structural, and that exact token-level likelihood is too strict a measure as many different reasoning paths can be correct as opposed to just one fixed correct answer. Our results therefore question the use of target-answer likelihood as a validation metric for LLM-based student simulators and advocate for more robust metrics for learning proxies."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="pit-vylomova-2026-context">
<titleInfo>
<title>When Context Sensitivity Isn’t Enough: An Argument Against Likelihood-Based Probes for Evaluating LLM-Simulated Students</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pagnarith</namePart>
<namePart type="family">Pit</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ekaterina</namePart>
<namePart type="family">Vylomova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kemal</namePart>
<namePart type="family">Kurniawan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyao</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarenne</namePart>
<namePart type="family">Wallbridge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fajri</namePart>
<namePart type="family">Koto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nouha</namePart>
<namePart type="family">Dziri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barbara</namePart>
<namePart type="family">Plank</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-484-2</identifier>
</relatedItem>
<abstract>Evaluating language models as student simulators currently focuses on whether their dialogue looks realistic, yet it is unclear whether these models actually respond to the quality of teaching the way human students do. However, this has not deterred prior works from using a model’s uncertainty, measured as the log-likelihood of a fixed correct answer, as a proxy for student behaviour. This approach has been used to simulate student responses, estimate question difficulty, and study “learning”. In this work, we test whether this proxy can detect sensitivity to instructional quality. We evaluate five language models on expert-rated tutoring dialogues under tightly controlled conditions. All models reliably distinguish relevant tutoring context from irrelevant context. However, this contextual sensitivity does not extend to pedagogical sensitivity: higher-quality instruction almost never increases the likelihood of the correct answer, and reasoning-oriented models sometimes show the opposite pattern. We argue that this failure is structural, and that exact token-level likelihood is too strict a measure as many different reasoning paths can be correct as opposed to just one fixed correct answer. Our results therefore question the use of target-answer likelihood as a validation metric for LLM-based student simulators and advocate for more robust metrics for learning proxies.</abstract>
<identifier type="citekey">pit-vylomova-2026-context</identifier>
<location>
<url>https://aclanthology.org/2026.pandora-1.7/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>135</start>
<end>148</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Context Sensitivity Isn’t Enough: An Argument Against Likelihood-Based Probes for Evaluating LLM-Simulated Students
%A Pit, Pagnarith
%A Vylomova, Ekaterina
%Y Xu, Shanshan
%Y T.Y.S.S., Santosh
%Y Kurniawan, Kemal
%Y Peng, Siyao
%Y Wallbridge, Sarenne
%Y Koto, Fajri
%Y Dziri, Nouha
%Y Plank, Barbara
%S Proceedings of the First Workshop on Pluralistic AI NLP: Diversity-aware, Sociotechnical, Responsible Alignment (PANDORA 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-484-2
%F pit-vylomova-2026-context
%X Evaluating language models as student simulators currently focuses on whether their dialogue looks realistic, yet it is unclear whether these models actually respond to the quality of teaching the way human students do. However, this has not deterred prior works from using a model’s uncertainty, measured as the log-likelihood of a fixed correct answer, as a proxy for student behaviour. This approach has been used to simulate student responses, estimate question difficulty, and study “learning”. In this work, we test whether this proxy can detect sensitivity to instructional quality. We evaluate five language models on expert-rated tutoring dialogues under tightly controlled conditions. All models reliably distinguish relevant tutoring context from irrelevant context. However, this contextual sensitivity does not extend to pedagogical sensitivity: higher-quality instruction almost never increases the likelihood of the correct answer, and reasoning-oriented models sometimes show the opposite pattern. We argue that this failure is structural, and that exact token-level likelihood is too strict a measure as many different reasoning paths can be correct as opposed to just one fixed correct answer. Our results therefore question the use of target-answer likelihood as a validation metric for LLM-based student simulators and advocate for more robust metrics for learning proxies.
%U https://aclanthology.org/2026.pandora-1.7/
%P 135-148
Markdown (Informal)
[When Context Sensitivity Isn’t Enough: An Argument Against Likelihood-Based Probes for Evaluating LLM-Simulated Students](https://aclanthology.org/2026.pandora-1.7/) (Pit & Vylomova, PANDORA 2026)
ACL