@inproceedings{truong-etal-2026-tutorial,
title = "A Tutorial on Measurement Science for Language Model Evaluation",
author = "Truong, Sang T. and
Wang, Serena Lutong and
Corona, Rodolfo and
Koyejo, Sanmi",
editor = "Okazaki, Naoaki and
Wang, Zhongqing",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics: Tutorial Abstracts",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-tutorials.3/",
pages = "20--26",
ISBN = "979-8-89176-443-9",
abstract = "As language models are increasingly optimized against the benchmarks used to evaluate them and deployed across diverse global populations, the assumptions behind evaluation are quietly breaking down. This tutorial focuses on a scientific approach for language model evaluation: a measurement-theoretic framework that treats LLM evaluation as a scientific measurement problem. We organize three hours around the classical structure of a measurement instrument and give NLP researchers usable tools for each part: (1) validity{--}when a benchmark score actually measures the capability it claims to, with emphasis on cross-cultural and cross-lingual transfer; (2) predictive measurement{--}inferring capability efficiently from sparse data via latent-variable models and cold-start prediction; and (3) design and incentives{--}why some benchmarks are gameable and what mechanisms improve robustness under explicit assumptions about evaluator access, developer objectives, and information leakage. The tutorial connects language model evaluation, latent-variable modeling, and mechanism design, three literatures that NLP researchers rarely encounter together, and equips the audience to build and interpret evaluations that remain trustworthy under deployment and strategic pressure."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="truong-etal-2026-tutorial">
<titleInfo>
<title>A Tutorial on Measurement Science for Language Model Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sang</namePart>
<namePart type="given">T</namePart>
<namePart type="family">Truong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Serena</namePart>
<namePart type="given">Lutong</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rodolfo</namePart>
<namePart type="family">Corona</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sanmi</namePart>
<namePart type="family">Koyejo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: Tutorial Abstracts</title>
</titleInfo>
<name type="personal">
<namePart type="given">Naoaki</namePart>
<namePart type="family">Okazaki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhongqing</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-443-9</identifier>
</relatedItem>
<abstract>As language models are increasingly optimized against the benchmarks used to evaluate them and deployed across diverse global populations, the assumptions behind evaluation are quietly breaking down. This tutorial focuses on a scientific approach for language model evaluation: a measurement-theoretic framework that treats LLM evaluation as a scientific measurement problem. We organize three hours around the classical structure of a measurement instrument and give NLP researchers usable tools for each part: (1) validity–when a benchmark score actually measures the capability it claims to, with emphasis on cross-cultural and cross-lingual transfer; (2) predictive measurement–inferring capability efficiently from sparse data via latent-variable models and cold-start prediction; and (3) design and incentives–why some benchmarks are gameable and what mechanisms improve robustness under explicit assumptions about evaluator access, developer objectives, and information leakage. The tutorial connects language model evaluation, latent-variable modeling, and mechanism design, three literatures that NLP researchers rarely encounter together, and equips the audience to build and interpret evaluations that remain trustworthy under deployment and strategic pressure.</abstract>
<identifier type="citekey">truong-etal-2026-tutorial</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-tutorials.3/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>20</start>
<end>26</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Tutorial on Measurement Science for Language Model Evaluation
%A Truong, Sang T.
%A Wang, Serena Lutong
%A Corona, Rodolfo
%A Koyejo, Sanmi
%Y Okazaki, Naoaki
%Y Wang, Zhongqing
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: Tutorial Abstracts
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-443-9
%F truong-etal-2026-tutorial
%X As language models are increasingly optimized against the benchmarks used to evaluate them and deployed across diverse global populations, the assumptions behind evaluation are quietly breaking down. This tutorial focuses on a scientific approach for language model evaluation: a measurement-theoretic framework that treats LLM evaluation as a scientific measurement problem. We organize three hours around the classical structure of a measurement instrument and give NLP researchers usable tools for each part: (1) validity–when a benchmark score actually measures the capability it claims to, with emphasis on cross-cultural and cross-lingual transfer; (2) predictive measurement–inferring capability efficiently from sparse data via latent-variable models and cold-start prediction; and (3) design and incentives–why some benchmarks are gameable and what mechanisms improve robustness under explicit assumptions about evaluator access, developer objectives, and information leakage. The tutorial connects language model evaluation, latent-variable modeling, and mechanism design, three literatures that NLP researchers rarely encounter together, and equips the audience to build and interpret evaluations that remain trustworthy under deployment and strategic pressure.
%U https://aclanthology.org/2026.aacl-tutorials.3/
%P 20-26
Markdown (Informal)
[A Tutorial on Measurement Science for Language Model Evaluation](https://aclanthology.org/2026.aacl-tutorials.3/) (Truong et al., AACL-IJCNLP 2026)
ACL
- Sang T. Truong, Serena Lutong Wang, Rodolfo Corona, and Sanmi Koyejo. 2026. A Tutorial on Measurement Science for Language Model Evaluation. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: Tutorial Abstracts, pages 20–26, Hengqin, China. Association for Computational Linguistics.