@inproceedings{yost-etal-2026-measure,
title = "Measure what Matters: Psychometric Evaluation of {AI} with Situational Judgment Tests",
author = "Yost, Alexandra and
Jain, Shreyans and
Raval, Shivam and
Corser, Grant C. and
Xu, Nina and
Roush, Allen G and
Hammack, Jacqueline and
Ziv, Ravid Shwartz and
Abdullah, Amir",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2453/",
pages = "52645--52750",
ISBN = "979-8-89176-462-0",
abstract = "Persona conditioning is widely used to steer large language model (LLM) behavior, but it is unclear whether it induces stable behavioral structure or superficial variation. We propose a framework to measure consistent behavioral tendencies using situational judgment tests (SJTs), multidimensional item response theory (MIRT), and structured synthetic personas, treating responses as observations of latent behavioral variables.\par Across large-scale SJT and persona datasets, we find that persona-conditioned behaviors are stable across runs, latent trait scores predict external benchmarks (e.g., TruthfulQA, EmoBench), and MIRT reveals consistent latent structure. We validate these results through human annotation, benchmark evaluation, and internal consistency analyses.\par We interpret these traits not as human personality, but as stable behavioral tendencies expressed across contexts. Our results show that scenario-based psychometric evaluation provides a more reliable alternative to classical self-report approaches for assessing LLM behavior, and we release datasets to support further study."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yost-etal-2026-measure">
<titleInfo>
<title>Measure what Matters: Psychometric Evaluation of AI with Situational Judgment Tests</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alexandra</namePart>
<namePart type="family">Yost</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shreyans</namePart>
<namePart type="family">Jain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shivam</namePart>
<namePart type="family">Raval</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Grant</namePart>
<namePart type="given">C</namePart>
<namePart type="family">Corser</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nina</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Allen</namePart>
<namePart type="given">G</namePart>
<namePart type="family">Roush</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jacqueline</namePart>
<namePart type="family">Hammack</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ravid</namePart>
<namePart type="given">Shwartz</namePart>
<namePart type="family">Ziv</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Amir</namePart>
<namePart type="family">Abdullah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Persona conditioning is widely used to steer large language model (LLM) behavior, but it is unclear whether it induces stable behavioral structure or superficial variation. We propose a framework to measure consistent behavioral tendencies using situational judgment tests (SJTs), multidimensional item response theory (MIRT), and structured synthetic personas, treating responses as observations of latent behavioral variables.\par Across large-scale SJT and persona datasets, we find that persona-conditioned behaviors are stable across runs, latent trait scores predict external benchmarks (e.g., TruthfulQA, EmoBench), and MIRT reveals consistent latent structure. We validate these results through human annotation, benchmark evaluation, and internal consistency analyses.\par We interpret these traits not as human personality, but as stable behavioral tendencies expressed across contexts. Our results show that scenario-based psychometric evaluation provides a more reliable alternative to classical self-report approaches for assessing LLM behavior, and we release datasets to support further study.</abstract>
<identifier type="citekey">yost-etal-2026-measure</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2453/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>52645</start>
<end>52750</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Measure what Matters: Psychometric Evaluation of AI with Situational Judgment Tests
%A Yost, Alexandra
%A Jain, Shreyans
%A Raval, Shivam
%A Corser, Grant C.
%A Xu, Nina
%A Roush, Allen G.
%A Hammack, Jacqueline
%A Ziv, Ravid Shwartz
%A Abdullah, Amir
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F yost-etal-2026-measure
%X Persona conditioning is widely used to steer large language model (LLM) behavior, but it is unclear whether it induces stable behavioral structure or superficial variation. We propose a framework to measure consistent behavioral tendencies using situational judgment tests (SJTs), multidimensional item response theory (MIRT), and structured synthetic personas, treating responses as observations of latent behavioral variables.\par Across large-scale SJT and persona datasets, we find that persona-conditioned behaviors are stable across runs, latent trait scores predict external benchmarks (e.g., TruthfulQA, EmoBench), and MIRT reveals consistent latent structure. We validate these results through human annotation, benchmark evaluation, and internal consistency analyses.\par We interpret these traits not as human personality, but as stable behavioral tendencies expressed across contexts. Our results show that scenario-based psychometric evaluation provides a more reliable alternative to classical self-report approaches for assessing LLM behavior, and we release datasets to support further study.
%U https://aclanthology.org/2026.findings-emnlp.2453/
%P 52645-52750
Markdown (Informal)
[Measure what Matters: Psychometric Evaluation of AI with Situational Judgment Tests](https://aclanthology.org/2026.findings-emnlp.2453/) (Yost et al., Findings 2026)
ACL
- Alexandra Yost, Shreyans Jain, Shivam Raval, Grant C. Corser, Nina Xu, Allen G Roush, Jacqueline Hammack, Ravid Shwartz Ziv, and Amir Abdullah. 2026. Measure what Matters: Psychometric Evaluation of AI with Situational Judgment Tests. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 52645–52750, Budapest, Hungary. Association for Computational Linguistics.