@inproceedings{hew-etal-2026-multiple,
title = "Between Multiple Choice and Open Response: Evaluating {LLM}s with Option-Free Deterministic Scoring",
author = "Hew, ZhongKen and
Yang, Sze Jue and
Chan, Chee Seng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.844/",
pages = "19582--19605",
ISBN = "979-8-89176-459-0",
abstract = "Benchmark scores are treated as evidence of LLM competence, but the same knowledge can yield different scores under different question formats. MCQs provide objective and scalable scoring, but their answer options can turn recall into recognition through answer-option scaffolding. Open-ended constructed response questions (CRQs) remove answer options, but their scores often depend on human or LLM judges and rubric interpretation, introducing judge-dependent scoring. This creates a trade-off between removing answer-option scaffolding and retaining objective scoring. We study whether this trade-off can be reduced through Constrained Structured Response Questions (CSRQs), a structured-response protocol that removes final answer options while preserving deterministic scoring. CSRQs use multi-select, ordering, and matching outputs as auditable answer spaces for exact verification. We evaluate CSRQs using converted knowledge benchmarks and MyCulture, which is, to our knowledge, the first Malaysia-centred cultural benchmark in Bahasa Malaysia, with parallel MCQ, CSRQ, and CRQ formats. Across benchmarks and model families, scores change when the evaluation instrument changes, i.e., scores drop after answer options are removed, and CRQ scores vary across judges. These findings suggest that benchmark scores should be interpreted as products of both model capability and the test used to measure it."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hew-etal-2026-multiple">
<titleInfo>
<title>Between Multiple Choice and Open Response: Evaluating LLMs with Option-Free Deterministic Scoring</title>
</titleInfo>
<name type="personal">
<namePart type="given">ZhongKen</namePart>
<namePart type="family">Hew</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sze</namePart>
<namePart type="given">Jue</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chee</namePart>
<namePart type="given">Seng</namePart>
<namePart type="family">Chan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Benchmark scores are treated as evidence of LLM competence, but the same knowledge can yield different scores under different question formats. MCQs provide objective and scalable scoring, but their answer options can turn recall into recognition through answer-option scaffolding. Open-ended constructed response questions (CRQs) remove answer options, but their scores often depend on human or LLM judges and rubric interpretation, introducing judge-dependent scoring. This creates a trade-off between removing answer-option scaffolding and retaining objective scoring. We study whether this trade-off can be reduced through Constrained Structured Response Questions (CSRQs), a structured-response protocol that removes final answer options while preserving deterministic scoring. CSRQs use multi-select, ordering, and matching outputs as auditable answer spaces for exact verification. We evaluate CSRQs using converted knowledge benchmarks and MyCulture, which is, to our knowledge, the first Malaysia-centred cultural benchmark in Bahasa Malaysia, with parallel MCQ, CSRQ, and CRQ formats. Across benchmarks and model families, scores change when the evaluation instrument changes, i.e., scores drop after answer options are removed, and CRQ scores vary across judges. These findings suggest that benchmark scores should be interpreted as products of both model capability and the test used to measure it.</abstract>
<identifier type="citekey">hew-etal-2026-multiple</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.844/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>19582</start>
<end>19605</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Between Multiple Choice and Open Response: Evaluating LLMs with Option-Free Deterministic Scoring
%A Hew, ZhongKen
%A Yang, Sze Jue
%A Chan, Chee Seng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F hew-etal-2026-multiple
%X Benchmark scores are treated as evidence of LLM competence, but the same knowledge can yield different scores under different question formats. MCQs provide objective and scalable scoring, but their answer options can turn recall into recognition through answer-option scaffolding. Open-ended constructed response questions (CRQs) remove answer options, but their scores often depend on human or LLM judges and rubric interpretation, introducing judge-dependent scoring. This creates a trade-off between removing answer-option scaffolding and retaining objective scoring. We study whether this trade-off can be reduced through Constrained Structured Response Questions (CSRQs), a structured-response protocol that removes final answer options while preserving deterministic scoring. CSRQs use multi-select, ordering, and matching outputs as auditable answer spaces for exact verification. We evaluate CSRQs using converted knowledge benchmarks and MyCulture, which is, to our knowledge, the first Malaysia-centred cultural benchmark in Bahasa Malaysia, with parallel MCQ, CSRQ, and CRQ formats. Across benchmarks and model families, scores change when the evaluation instrument changes, i.e., scores drop after answer options are removed, and CRQ scores vary across judges. These findings suggest that benchmark scores should be interpreted as products of both model capability and the test used to measure it.
%U https://aclanthology.org/2026.emnlp-main.844/
%P 19582-19605
Markdown (Informal)
[Between Multiple Choice and Open Response: Evaluating LLMs with Option-Free Deterministic Scoring](https://aclanthology.org/2026.emnlp-main.844/) (Hew et al., EMNLP 2026)
ACL