@inproceedings{basu-2026-prompting,
title = "Prompting-Based Evaluation of Compact Language Models for {A}ustralian Mental Health Ethics",
author = "Basu, Subhrajyoti",
editor = "Wang, Zimu and
Jiang, Yiwen and
Shen, Yaling and
Zhao, Xiangyu and
Fong, Stephanie and
Sun, Xin and
Zhou, Yi and
Ji, Shaoxiong and
Desimone, Robert and
Nelson, Barnaby and
Dwyer, Dominic and
Ge, Zongyuan",
booktitle = "Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy ({M}ulti{P}syche)",
month = nov,
year = "2026",
address = "Hengqin, Zhuhai, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.multipsyche-1.16/",
pages = "186--193",
ISBN = "979-8-89176-448-4",
abstract = "Large language models are increasingly being explored for mental health applications, making reliable evaluation of their ethical behavior particularly important. However, professional ethics in psychology and psychiatry are jurisdiction-dependent and often require selecting multiple applicable principles rather than identifying a single correct response. The PsychEthicsEval shared task addresses this challenge through principle-grounded evaluation based on Australian mental health ethics. We investigate whether a compact language model can perform effectively in this setting without task-specific training. Our approach uses the frozen Qwen3.5-4B model with 4-bit quantized, prompting-only inference, combined with explicit Australian-jurisdiction grounding, answer-cardinality instructions, and deterministic answer extraction. In the original run on the 100-item labeled Phase 1 set, P2 achieves 70.00{\%} exact match (EM) and 84.50{\%} partial credit (PC), with 100.00{\%} EM/PC on single-answer questions compared with 38.78{\%} EM and 68.37{\%} PC on multiple-answer questions. On the official Phase 2 evaluation of 282 held-out items, it achieves 65.20{\%} EM and 77.50{\%} PC (reported 95{\%} confidence interval for PC: 73.40-81.20). A post-submission prompt-sensitivity analysis on Phase 1 found that an alternative wording reached 86.00{\%} EM and 92.00{\%} PC, while an option-wise diagnostic prompt performed worse overall. These exploratory results provide a reproducible low-resource prompting-only baseline and highlight the challenge of complete answer-set identification; alternative prompts were not evaluated on Phase 2."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="basu-2026-prompting">
<titleInfo>
<title>Prompting-Based Evaluation of Compact Language Models for Australian Mental Health Ethics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Subhrajyoti</namePart>
<namePart type="family">Basu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiwen</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaling</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiangyu</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Stephanie</namePart>
<namePart type="family">Fong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xin</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shaoxiong</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Robert</namePart>
<namePart type="family">Desimone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Barnaby</namePart>
<namePart type="family">Nelson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dominic</namePart>
<namePart type="family">Dwyer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zongyuan</namePart>
<namePart type="family">Ge</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, Zhuhai, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-448-4</identifier>
</relatedItem>
<abstract>Large language models are increasingly being explored for mental health applications, making reliable evaluation of their ethical behavior particularly important. However, professional ethics in psychology and psychiatry are jurisdiction-dependent and often require selecting multiple applicable principles rather than identifying a single correct response. The PsychEthicsEval shared task addresses this challenge through principle-grounded evaluation based on Australian mental health ethics. We investigate whether a compact language model can perform effectively in this setting without task-specific training. Our approach uses the frozen Qwen3.5-4B model with 4-bit quantized, prompting-only inference, combined with explicit Australian-jurisdiction grounding, answer-cardinality instructions, and deterministic answer extraction. In the original run on the 100-item labeled Phase 1 set, P2 achieves 70.00% exact match (EM) and 84.50% partial credit (PC), with 100.00% EM/PC on single-answer questions compared with 38.78% EM and 68.37% PC on multiple-answer questions. On the official Phase 2 evaluation of 282 held-out items, it achieves 65.20% EM and 77.50% PC (reported 95% confidence interval for PC: 73.40-81.20). A post-submission prompt-sensitivity analysis on Phase 1 found that an alternative wording reached 86.00% EM and 92.00% PC, while an option-wise diagnostic prompt performed worse overall. These exploratory results provide a reproducible low-resource prompting-only baseline and highlight the challenge of complete answer-set identification; alternative prompts were not evaluated on Phase 2.</abstract>
<identifier type="citekey">basu-2026-prompting</identifier>
<location>
<url>https://aclanthology.org/2026.multipsyche-1.16/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>186</start>
<end>193</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Prompting-Based Evaluation of Compact Language Models for Australian Mental Health Ethics
%A Basu, Subhrajyoti
%Y Wang, Zimu
%Y Jiang, Yiwen
%Y Shen, Yaling
%Y Zhao, Xiangyu
%Y Fong, Stephanie
%Y Sun, Xin
%Y Zhou, Yi
%Y Ji, Shaoxiong
%Y Desimone, Robert
%Y Nelson, Barnaby
%Y Dwyer, Dominic
%Y Ge, Zongyuan
%S Proceedings of the 1st Workshop on Multimodal, Multilingual, and Multicultural Mental Health and Psychotherapy (MultiPsyche)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, Zhuhai, China
%@ 979-8-89176-448-4
%F basu-2026-prompting
%X Large language models are increasingly being explored for mental health applications, making reliable evaluation of their ethical behavior particularly important. However, professional ethics in psychology and psychiatry are jurisdiction-dependent and often require selecting multiple applicable principles rather than identifying a single correct response. The PsychEthicsEval shared task addresses this challenge through principle-grounded evaluation based on Australian mental health ethics. We investigate whether a compact language model can perform effectively in this setting without task-specific training. Our approach uses the frozen Qwen3.5-4B model with 4-bit quantized, prompting-only inference, combined with explicit Australian-jurisdiction grounding, answer-cardinality instructions, and deterministic answer extraction. In the original run on the 100-item labeled Phase 1 set, P2 achieves 70.00% exact match (EM) and 84.50% partial credit (PC), with 100.00% EM/PC on single-answer questions compared with 38.78% EM and 68.37% PC on multiple-answer questions. On the official Phase 2 evaluation of 282 held-out items, it achieves 65.20% EM and 77.50% PC (reported 95% confidence interval for PC: 73.40-81.20). A post-submission prompt-sensitivity analysis on Phase 1 found that an alternative wording reached 86.00% EM and 92.00% PC, while an option-wise diagnostic prompt performed worse overall. These exploratory results provide a reproducible low-resource prompting-only baseline and highlight the challenge of complete answer-set identification; alternative prompts were not evaluated on Phase 2.
%U https://aclanthology.org/2026.multipsyche-1.16/
%P 186-193
Markdown (Informal)
[Prompting-Based Evaluation of Compact Language Models for Australian Mental Health Ethics](https://aclanthology.org/2026.multipsyche-1.16/) (Basu, MultiPsyche 2026)
ACL