@inproceedings{xiaolongwang-etal-2026-beyond,
title = "Beyond Task-Based Evaluation: Toward Situated Benchmarks for Educational {LLM}s",
author = "Xiaolongwang and
Xu, Tianyi and
Zou, Rui and
Zuo, Jiawen and
Zhang, Mingzi and
Li, Keqian and
Zhao, Zhe and
Wei, Ye and
Zhou, Aimin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1503/",
pages = "32011--32023",
ISBN = "979-8-89176-462-0",
abstract = "This paper argues that evaluation of educational LLMs should move beyond decontextualized task-based benchmarks toward situated benchmarks centered on pedagogical action. Prior work in human-computer interaction and the learning sciences has long emphasized situated use, user context, and interaction processes; our focus is narrower and complementary: how these concerns should reshape benchmark design for educational LLMs. Existing evaluations already cover question generation, knowledge answering, error diagnosis, feedback generation, and teaching analysis, but their dominant unit remains the single input-output episode, making it difficult to compare whether a model genuinely supports learning progress, instructional judgment, or subsequent revision in context. Educational activity is processual, context-dependent, and development-oriented, so explanations, feedback, and hints can only be evaluated properly in relation to learner state, interaction history, and pedagogical goals. We review a landscape still dominated by task-based paradigms and argue, from educational conceptions, educational values, and research methodology, that next-generation educational benchmarks should make pedagogical function in context a core target. We further propose layered, scalable, and benchmarkable situated protocols built through simulation-first development and follow-up empirical calibration, extending evaluation beyond task quality to contextual fit and downstream consequences for learning and teaching decisions."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xiaolongwang-etal-2026-beyond">
<titleInfo>
<title>Beyond Task-Based Evaluation: Toward Situated Benchmarks for Educational LLMs</title>
</titleInfo>
<name>
<namePart>Xiaolongwang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianyi</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Zou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiawen</namePart>
<namePart type="family">Zuo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mingzi</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keqian</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhe</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ye</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aimin</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>This paper argues that evaluation of educational LLMs should move beyond decontextualized task-based benchmarks toward situated benchmarks centered on pedagogical action. Prior work in human-computer interaction and the learning sciences has long emphasized situated use, user context, and interaction processes; our focus is narrower and complementary: how these concerns should reshape benchmark design for educational LLMs. Existing evaluations already cover question generation, knowledge answering, error diagnosis, feedback generation, and teaching analysis, but their dominant unit remains the single input-output episode, making it difficult to compare whether a model genuinely supports learning progress, instructional judgment, or subsequent revision in context. Educational activity is processual, context-dependent, and development-oriented, so explanations, feedback, and hints can only be evaluated properly in relation to learner state, interaction history, and pedagogical goals. We review a landscape still dominated by task-based paradigms and argue, from educational conceptions, educational values, and research methodology, that next-generation educational benchmarks should make pedagogical function in context a core target. We further propose layered, scalable, and benchmarkable situated protocols built through simulation-first development and follow-up empirical calibration, extending evaluation beyond task quality to contextual fit and downstream consequences for learning and teaching decisions.</abstract>
<identifier type="citekey">xiaolongwang-etal-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1503/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>32011</start>
<end>32023</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Task-Based Evaluation: Toward Situated Benchmarks for Educational LLMs
%A Xu, Tianyi
%A Zou, Rui
%A Zuo, Jiawen
%A Zhang, Mingzi
%A Li, Keqian
%A Zhao, Zhe
%A Wei, Ye
%A Zhou, Aimin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%A Xiaolongwang
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F xiaolongwang-etal-2026-beyond
%X This paper argues that evaluation of educational LLMs should move beyond decontextualized task-based benchmarks toward situated benchmarks centered on pedagogical action. Prior work in human-computer interaction and the learning sciences has long emphasized situated use, user context, and interaction processes; our focus is narrower and complementary: how these concerns should reshape benchmark design for educational LLMs. Existing evaluations already cover question generation, knowledge answering, error diagnosis, feedback generation, and teaching analysis, but their dominant unit remains the single input-output episode, making it difficult to compare whether a model genuinely supports learning progress, instructional judgment, or subsequent revision in context. Educational activity is processual, context-dependent, and development-oriented, so explanations, feedback, and hints can only be evaluated properly in relation to learner state, interaction history, and pedagogical goals. We review a landscape still dominated by task-based paradigms and argue, from educational conceptions, educational values, and research methodology, that next-generation educational benchmarks should make pedagogical function in context a core target. We further propose layered, scalable, and benchmarkable situated protocols built through simulation-first development and follow-up empirical calibration, extending evaluation beyond task quality to contextual fit and downstream consequences for learning and teaching decisions.
%U https://aclanthology.org/2026.findings-emnlp.1503/
%P 32011-32023
Markdown (Informal)
[Beyond Task-Based Evaluation: Toward Situated Benchmarks for Educational LLMs](https://aclanthology.org/2026.findings-emnlp.1503/) (Xiaolongwang et al., Findings 2026)
ACL
- Xiaolongwang, Tianyi Xu, Rui Zou, Jiawen Zuo, Mingzi Zhang, Keqian Li, Zhe Zhao, Ye Wei, and Aimin Zhou. 2026. Beyond Task-Based Evaluation: Toward Situated Benchmarks for Educational LLMs. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 32011–32023, Budapest, Hungary. Association for Computational Linguistics.