@inproceedings{shen-etal-2026-overview,
title = "Overview of the {P}sych{E}thics{E}val Shared Task at {M}ulti{P}syche 2026: Evaluating {LLM}s Against {A}ustralian Mental Health Ethics",
author = "Shen, Yaling and
Wang, Zimu and
Fong, Stephanie and
Jiang, Yiwen and
Tang, Feilong and
Xu, Qingyang and
Zhao, Xiangyu and
Xu, Zhongxing and
Liu, Jiahe and
Hu, Jinpeng and
Ge, Zongyuan and
Dwyer, Dominic",
editor = "Wang, Zimu and
Jiang, Yiwen and
Shen, Yaling and
Zhao, Xiangyu and
Fong, Stephanie and
Sun, Xin and
Zhou, Yi and
Ji, Shaoxiong and
Desimone, Robert and
Nelson, Barnaby and
Dwyer, Dominic and
Ge, Zongyuan",
booktitle = "Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy ({M}ulti{P}syche)",
month = nov,
year = "2026",
address = "Hengqin, Zhuhai, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.multipsyche-1.12/",
pages = "159--163",
ISBN = "979-8-89176-448-4",
abstract = "We present PsychEthicsEval, a shared task that evaluates the alignment of large language models (LLMs) with Australian professional ethics guidelines in psychology and psychiatry. Building on PsychEthicsBench, the task comprises two complementary tracks: Multiple-Choice Questions (MCQ) assessing explicit ethical knowledge and Open-Ended Questions (OEQ) assessing ethical behavior in free-form responses. MCQ systems are ranked by Partial Credit (PC), while OEQ ethical rates are reported separately under rule-classifier and LLM-as-a-judge protocols. Four teams submitted final systems, with three systems evaluated in each track. AI4Health achieves the highest MCQ PC score of 0.832 and leads the rule-classifier OEQ evaluation, whereas XJTLU-EthicsPrompt leads under the LLM judge. Diagnostic analysis identifies judgments about professional identity and qualifications as the main source of the score gap between these two systems under the LLM judge, while jurisdictional applicability is a major source of score loss for group11 under the rule-classifier protocol, showing that comparisons of systems in open-ended ethical evaluation depend on how professional ethical criteria are applied."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shen-etal-2026-overview">
<titleInfo>
<title>Overview of the PsychEthicsEval Shared Task at MultiPsyche 2026: Evaluating LLMs Against Australian Mental Health Ethics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yaling</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Stephanie</namePart>
<namePart type="family">Fong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiwen</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Feilong</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qingyang</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiangyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhongxing</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiahe</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinpeng</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zongyuan</namePart>
<namePart type="family">Ge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dominic</namePart>
<namePart type="family">Dwyer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiwen</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaling</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiangyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Stephanie</namePart>
<namePart type="family">Fong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaoxiong</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="family">Desimone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barnaby</namePart>
<namePart type="family">Nelson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dominic</namePart>
<namePart type="family">Dwyer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zongyuan</namePart>
<namePart type="family">Ge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, Zhuhai, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-448-4</identifier>
</relatedItem>
<abstract>We present PsychEthicsEval, a shared task that evaluates the alignment of large language models (LLMs) with Australian professional ethics guidelines in psychology and psychiatry. Building on PsychEthicsBench, the task comprises two complementary tracks: Multiple-Choice Questions (MCQ) assessing explicit ethical knowledge and Open-Ended Questions (OEQ) assessing ethical behavior in free-form responses. MCQ systems are ranked by Partial Credit (PC), while OEQ ethical rates are reported separately under rule-classifier and LLM-as-a-judge protocols. Four teams submitted final systems, with three systems evaluated in each track. AI4Health achieves the highest MCQ PC score of 0.832 and leads the rule-classifier OEQ evaluation, whereas XJTLU-EthicsPrompt leads under the LLM judge. Diagnostic analysis identifies judgments about professional identity and qualifications as the main source of the score gap between these two systems under the LLM judge, while jurisdictional applicability is a major source of score loss for group11 under the rule-classifier protocol, showing that comparisons of systems in open-ended ethical evaluation depend on how professional ethical criteria are applied.</abstract>
<identifier type="citekey">shen-etal-2026-overview</identifier>
<location>
<url>https://aclanthology.org/2026.multipsyche-1.12/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>159</start>
<end>163</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Overview of the PsychEthicsEval Shared Task at MultiPsyche 2026: Evaluating LLMs Against Australian Mental Health Ethics
%A Shen, Yaling
%A Wang, Zimu
%A Fong, Stephanie
%A Jiang, Yiwen
%A Tang, Feilong
%A Xu, Qingyang
%A Zhao, Xiangyu
%A Xu, Zhongxing
%A Liu, Jiahe
%A Hu, Jinpeng
%A Ge, Zongyuan
%A Dwyer, Dominic
%Y Wang, Zimu
%Y Jiang, Yiwen
%Y Shen, Yaling
%Y Zhao, Xiangyu
%Y Fong, Stephanie
%Y Sun, Xin
%Y Zhou, Yi
%Y Ji, Shaoxiong
%Y Desimone, Robert
%Y Nelson, Barnaby
%Y Dwyer, Dominic
%Y Ge, Zongyuan
%S Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, Zhuhai, China
%@ 979-8-89176-448-4
%F shen-etal-2026-overview
%X We present PsychEthicsEval, a shared task that evaluates the alignment of large language models (LLMs) with Australian professional ethics guidelines in psychology and psychiatry. Building on PsychEthicsBench, the task comprises two complementary tracks: Multiple-Choice Questions (MCQ) assessing explicit ethical knowledge and Open-Ended Questions (OEQ) assessing ethical behavior in free-form responses. MCQ systems are ranked by Partial Credit (PC), while OEQ ethical rates are reported separately under rule-classifier and LLM-as-a-judge protocols. Four teams submitted final systems, with three systems evaluated in each track. AI4Health achieves the highest MCQ PC score of 0.832 and leads the rule-classifier OEQ evaluation, whereas XJTLU-EthicsPrompt leads under the LLM judge. Diagnostic analysis identifies judgments about professional identity and qualifications as the main source of the score gap between these two systems under the LLM judge, while jurisdictional applicability is a major source of score loss for group11 under the rule-classifier protocol, showing that comparisons of systems in open-ended ethical evaluation depend on how professional ethical criteria are applied.
%U https://aclanthology.org/2026.multipsyche-1.12/
%P 159-163
Markdown (Informal)
[Overview of the PsychEthicsEval Shared Task at MultiPsyche 2026: Evaluating LLMs Against Australian Mental Health Ethics](https://aclanthology.org/2026.multipsyche-1.12/) (Shen et al., MultiPsyche 2026)
ACL
- Yaling Shen, Zimu Wang, Stephanie Fong, Yiwen Jiang, Feilong Tang, Qingyang Xu, Xiangyu Zhao, Zhongxing Xu, Jiahe Liu, Jinpeng Hu, Zongyuan Ge, and Dominic Dwyer. 2026. Overview of the PsychEthicsEval Shared Task at MultiPsyche 2026: Evaluating LLMs Against Australian Mental Health Ethics. In Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche), pages 159–163, Hengqin, Zhuhai, China. Association for Computational Linguistics.