@inproceedings{khoi-hoang-etal-2026-psi,
title = "{PSI}-Bench: Interpretable and Clinically Meaningful Evaluation of Depression Patient Simulators",
author = {Khoi Hoang, Nguyen and
Mehri, Shuhaib and
Hsu, Tse-An and
Sun, Yi-Jyun and
Truong, Quynh Xuan Nguyen and
Doan, Khoa D and
Hakkani-T{\"u}r, Dilek},
editor = "Wang, Zimu and
Jiang, Yiwen and
Shen, Yaling and
Zhao, Xiangyu and
Fong, Stephanie and
Sun, Xin and
Zhou, Yi and
Ji, Shaoxiong and
Desimone, Robert and
Nelson, Barnaby and
Dwyer, Dominic and
Ge, Zongyuan",
booktitle = "Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy ({M}ulti{P}syche)",
month = nov,
year = "2026",
address = "Hengqin, Zhuhai, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.multipsyche-1.10/",
pages = "118--144",
ISBN = "979-8-89176-448-4",
abstract = "Patient simulators are gaining traction in mental health training by providing scalable exposure to complex and sensitive patient interactions. Simulating depressed patients is challenging, as safety constraints and high patient variability complicate simulations and underscore the need for simulators that capture diverse and realistic patient behaviors. However, existing evaluations heavily rely on LLM-judges with poorly specified prompts and do not assess behavioral diversity. We introduce PSI-Bench, an automatic evaluation framework that provides interpretable, clinically meaningful diagnostics of depression patient simulator behavior across turn-, dialogue-, and population-level dimensions. Using PSI-Bench, we benchmark seven LLMs across two simulator frameworks and find that simulators produce overly long, lexically diverse responses, show reduced variability, and move through therapeutic stages and toward positive valence too quickly. We also show that the simulation framework has a larger impact on fidelity than the model scale. Results from a human study demonstrate that our benchmark is strongly aligned with judgments of mental health professionals. Our work reveals key limitations of current depression patient simulators and provides an interpretable, extensible benchmark to guide future simulator design and evaluation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="khoi-hoang-etal-2026-psi">
<titleInfo>
<title>PSI-Bench: Interpretable and Clinically Meaningful Evaluation of Depression Patient Simulators</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nguyen</namePart>
<namePart type="family">Khoi Hoang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuhaib</namePart>
<namePart type="family">Mehri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tse-An</namePart>
<namePart type="family">Hsu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi-Jyun</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Quynh</namePart>
<namePart type="given">Xuan</namePart>
<namePart type="given">Nguyen</namePart>
<namePart type="family">Truong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Khoa</namePart>
<namePart type="given">D</namePart>
<namePart type="family">Doan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dilek</namePart>
<namePart type="family">Hakkani-Tür</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiwen</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaling</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiangyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Stephanie</namePart>
<namePart type="family">Fong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaoxiong</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="family">Desimone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barnaby</namePart>
<namePart type="family">Nelson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dominic</namePart>
<namePart type="family">Dwyer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zongyuan</namePart>
<namePart type="family">Ge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, Zhuhai, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-448-4</identifier>
</relatedItem>
<abstract>Patient simulators are gaining traction in mental health training by providing scalable exposure to complex and sensitive patient interactions. Simulating depressed patients is challenging, as safety constraints and high patient variability complicate simulations and underscore the need for simulators that capture diverse and realistic patient behaviors. However, existing evaluations heavily rely on LLM-judges with poorly specified prompts and do not assess behavioral diversity. We introduce PSI-Bench, an automatic evaluation framework that provides interpretable, clinically meaningful diagnostics of depression patient simulator behavior across turn-, dialogue-, and population-level dimensions. Using PSI-Bench, we benchmark seven LLMs across two simulator frameworks and find that simulators produce overly long, lexically diverse responses, show reduced variability, and move through therapeutic stages and toward positive valence too quickly. We also show that the simulation framework has a larger impact on fidelity than the model scale. Results from a human study demonstrate that our benchmark is strongly aligned with judgments of mental health professionals. Our work reveals key limitations of current depression patient simulators and provides an interpretable, extensible benchmark to guide future simulator design and evaluation.</abstract>
<identifier type="citekey">khoi-hoang-etal-2026-psi</identifier>
<location>
<url>https://aclanthology.org/2026.multipsyche-1.10/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>118</start>
<end>144</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PSI-Bench: Interpretable and Clinically Meaningful Evaluation of Depression Patient Simulators
%A Khoi Hoang, Nguyen
%A Mehri, Shuhaib
%A Hsu, Tse-An
%A Sun, Yi-Jyun
%A Truong, Quynh Xuan Nguyen
%A Doan, Khoa D.
%A Hakkani-Tür, Dilek
%Y Wang, Zimu
%Y Jiang, Yiwen
%Y Shen, Yaling
%Y Zhao, Xiangyu
%Y Fong, Stephanie
%Y Sun, Xin
%Y Zhou, Yi
%Y Ji, Shaoxiong
%Y Desimone, Robert
%Y Nelson, Barnaby
%Y Dwyer, Dominic
%Y Ge, Zongyuan
%S Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, Zhuhai, China
%@ 979-8-89176-448-4
%F khoi-hoang-etal-2026-psi
%X Patient simulators are gaining traction in mental health training by providing scalable exposure to complex and sensitive patient interactions. Simulating depressed patients is challenging, as safety constraints and high patient variability complicate simulations and underscore the need for simulators that capture diverse and realistic patient behaviors. However, existing evaluations heavily rely on LLM-judges with poorly specified prompts and do not assess behavioral diversity. We introduce PSI-Bench, an automatic evaluation framework that provides interpretable, clinically meaningful diagnostics of depression patient simulator behavior across turn-, dialogue-, and population-level dimensions. Using PSI-Bench, we benchmark seven LLMs across two simulator frameworks and find that simulators produce overly long, lexically diverse responses, show reduced variability, and move through therapeutic stages and toward positive valence too quickly. We also show that the simulation framework has a larger impact on fidelity than the model scale. Results from a human study demonstrate that our benchmark is strongly aligned with judgments of mental health professionals. Our work reveals key limitations of current depression patient simulators and provides an interpretable, extensible benchmark to guide future simulator design and evaluation.
%U https://aclanthology.org/2026.multipsyche-1.10/
%P 118-144
Markdown (Informal)
[PSI-Bench: Interpretable and Clinically Meaningful Evaluation of Depression Patient Simulators](https://aclanthology.org/2026.multipsyche-1.10/) (Khoi Hoang et al., MultiPsyche 2026)
ACL
- Nguyen Khoi Hoang, Shuhaib Mehri, Tse-An Hsu, Yi-Jyun Sun, Quynh Xuan Nguyen Truong, Khoa D Doan, and Dilek Hakkani-Tür. 2026. PSI-Bench: Interpretable and Clinically Meaningful Evaluation of Depression Patient Simulators. In Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche), pages 118–144, Hengqin, Zhuhai, China. Association for Computational Linguistics.