@inproceedings{park-etal-2026-accuracy,
title = "When Accuracy Aligns and Fails: Diagnosing Human{--}{LLM} Response Alignment at Population Scale",
author = "Park, Seoyoon and
Choi, Hyeji and
An, Subin and
Wang, Xiaonan and
Kim, Heejae and
Kang, Joeun and
Kim, Hansaem",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1944/",
pages = "44234--44254",
ISBN = "979-8-89176-459-0",
abstract = "We propose CAPE, a Choice-Distribution Alignment and Performance Evaluation framework for diagnosing human{--}LLM response alignment using population-scale human choice distributions. CAPE separates outcome-level difficulty alignment from response-distribution alignment with four metrics: OA, MAE, PA, and DA. We apply CAPE to 609 Korean CSAT items with item-level response distributions from approximately 300K examinees per item, and evaluate 16 LLMs. Results show that model capability is the strongest predictor of alignment, with accuracy closely tied to both full-choice and distractor-only alignment. However, group-level scores are affected by aggregation size and model composition, and neither reasoning augmentation nor Korean adaptation yields consistent residual advantages. Bloom-level analyses further suggest that capability-linked alignment varies across item types. CAPE thus enables evaluation beyond accuracy by diagnosing how model success and failure patterns relate to human population responses."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="park-etal-2026-accuracy">
<titleInfo>
<title>When Accuracy Aligns and Fails: Diagnosing Human–LLM Response Alignment at Population Scale</title>
</titleInfo>
<name type="personal">
<namePart type="given">Seoyoon</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hyeji</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Subin</namePart>
<namePart type="family">An</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaonan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Heejae</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Joeun</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hansaem</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>We propose CAPE, a Choice-Distribution Alignment and Performance Evaluation framework for diagnosing human–LLM response alignment using population-scale human choice distributions. CAPE separates outcome-level difficulty alignment from response-distribution alignment with four metrics: OA, MAE, PA, and DA. We apply CAPE to 609 Korean CSAT items with item-level response distributions from approximately 300K examinees per item, and evaluate 16 LLMs. Results show that model capability is the strongest predictor of alignment, with accuracy closely tied to both full-choice and distractor-only alignment. However, group-level scores are affected by aggregation size and model composition, and neither reasoning augmentation nor Korean adaptation yields consistent residual advantages. Bloom-level analyses further suggest that capability-linked alignment varies across item types. CAPE thus enables evaluation beyond accuracy by diagnosing how model success and failure patterns relate to human population responses.</abstract>
<identifier type="citekey">park-etal-2026-accuracy</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1944/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44234</start>
<end>44254</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T When Accuracy Aligns and Fails: Diagnosing Human–LLM Response Alignment at Population Scale
%A Park, Seoyoon
%A Choi, Hyeji
%A An, Subin
%A Wang, Xiaonan
%A Kim, Heejae
%A Kang, Joeun
%A Kim, Hansaem
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F park-etal-2026-accuracy
%X We propose CAPE, a Choice-Distribution Alignment and Performance Evaluation framework for diagnosing human–LLM response alignment using population-scale human choice distributions. CAPE separates outcome-level difficulty alignment from response-distribution alignment with four metrics: OA, MAE, PA, and DA. We apply CAPE to 609 Korean CSAT items with item-level response distributions from approximately 300K examinees per item, and evaluate 16 LLMs. Results show that model capability is the strongest predictor of alignment, with accuracy closely tied to both full-choice and distractor-only alignment. However, group-level scores are affected by aggregation size and model composition, and neither reasoning augmentation nor Korean adaptation yields consistent residual advantages. Bloom-level analyses further suggest that capability-linked alignment varies across item types. CAPE thus enables evaluation beyond accuracy by diagnosing how model success and failure patterns relate to human population responses.
%U https://aclanthology.org/2026.emnlp-main.1944/
%P 44234-44254
Markdown (Informal)
[When Accuracy Aligns and Fails: Diagnosing Human–LLM Response Alignment at Population Scale](https://aclanthology.org/2026.emnlp-main.1944/) (Park et al., EMNLP 2026)
ACL