@inproceedings{dutta-etal-2026-pulse,
title = "{PULSE}-{CARE}: A Measurement-Validity Audit of Simulated-Patient Clinical Agent Evaluation",
author = "Dutta, Srimonti and
Mishra, Dinky and
Singh, Aditi and
Moharir, Akshata Kishore",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.266/",
pages = "5904--5923",
ISBN = "979-8-89176-439-2",
abstract = "Simulated-patient benchmarks are widely used to evaluate clinical AI agents, yet the patient simulator itself can alter the evidence available to the agent and therefore the conclusions drawn from the benchmark. We introduce PULSE-CARE, a measurement-validity audit for simulated-patient clinical-agent evaluation that quantifies sensitivity to patient simulator behavior, agent architecture, simulator backbone, scorer choice, and demographic framing. Across 24 synthetic longitudinal cases, four dynamic simulator variants, three agent models, and two architectures, the mean per-case score range across simulators is 0.25 (95{\%} CI: 0.21-0.29), and escalation decisions differ across simulator conditions for 60{\%} of agent-case pairs. A 392-trajectory repeated-sampling analysis shows that simulator effects persist after pooling repeated draws, while also revealing substantial within-policy stochastic variation. Nominal model rank reversals occur in 17{\%} of comparisons but disappear under a 0.05 practical-difference margin. Demographic counterfactuals change escalation in 26{\%} of matched pairs, essentially the same rate as the same-input repeated-sampling control, providing no evidence of a demographic effect in this audit. PULSE-CARE combines simulator-sensitivity metrics, EFI composite indices, Patient Simulator Cards, and an Audit Report template to make the measurement conditions behind simulated-patient benchmark results explicit. Our results show that clinical-agent performance reported from a single patient simulator should be interpreted as conditional on the simulator and evaluation pipeline used to obtain it."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dutta-etal-2026-pulse">
<titleInfo>
<title>PULSE-CARE: A Measurement-Validity Audit of Simulated-Patient Clinical Agent Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Srimonti</namePart>
<namePart type="family">Dutta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dinky</namePart>
<namePart type="family">Mishra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aditi</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akshata</namePart>
<namePart type="given">Kishore</namePart>
<namePart type="family">Moharir</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Simulated-patient benchmarks are widely used to evaluate clinical AI agents, yet the patient simulator itself can alter the evidence available to the agent and therefore the conclusions drawn from the benchmark. We introduce PULSE-CARE, a measurement-validity audit for simulated-patient clinical-agent evaluation that quantifies sensitivity to patient simulator behavior, agent architecture, simulator backbone, scorer choice, and demographic framing. Across 24 synthetic longitudinal cases, four dynamic simulator variants, three agent models, and two architectures, the mean per-case score range across simulators is 0.25 (95% CI: 0.21-0.29), and escalation decisions differ across simulator conditions for 60% of agent-case pairs. A 392-trajectory repeated-sampling analysis shows that simulator effects persist after pooling repeated draws, while also revealing substantial within-policy stochastic variation. Nominal model rank reversals occur in 17% of comparisons but disappear under a 0.05 practical-difference margin. Demographic counterfactuals change escalation in 26% of matched pairs, essentially the same rate as the same-input repeated-sampling control, providing no evidence of a demographic effect in this audit. PULSE-CARE combines simulator-sensitivity metrics, EFI composite indices, Patient Simulator Cards, and an Audit Report template to make the measurement conditions behind simulated-patient benchmark results explicit. Our results show that clinical-agent performance reported from a single patient simulator should be interpreted as conditional on the simulator and evaluation pipeline used to obtain it.</abstract>
<identifier type="citekey">dutta-etal-2026-pulse</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.266/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>5904</start>
<end>5923</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PULSE-CARE: A Measurement-Validity Audit of Simulated-Patient Clinical Agent Evaluation
%A Dutta, Srimonti
%A Mishra, Dinky
%A Singh, Aditi
%A Moharir, Akshata Kishore
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F dutta-etal-2026-pulse
%X Simulated-patient benchmarks are widely used to evaluate clinical AI agents, yet the patient simulator itself can alter the evidence available to the agent and therefore the conclusions drawn from the benchmark. We introduce PULSE-CARE, a measurement-validity audit for simulated-patient clinical-agent evaluation that quantifies sensitivity to patient simulator behavior, agent architecture, simulator backbone, scorer choice, and demographic framing. Across 24 synthetic longitudinal cases, four dynamic simulator variants, three agent models, and two architectures, the mean per-case score range across simulators is 0.25 (95% CI: 0.21-0.29), and escalation decisions differ across simulator conditions for 60% of agent-case pairs. A 392-trajectory repeated-sampling analysis shows that simulator effects persist after pooling repeated draws, while also revealing substantial within-policy stochastic variation. Nominal model rank reversals occur in 17% of comparisons but disappear under a 0.05 practical-difference margin. Demographic counterfactuals change escalation in 26% of matched pairs, essentially the same rate as the same-input repeated-sampling control, providing no evidence of a demographic effect in this audit. PULSE-CARE combines simulator-sensitivity metrics, EFI composite indices, Patient Simulator Cards, and an Audit Report template to make the measurement conditions behind simulated-patient benchmark results explicit. Our results show that clinical-agent performance reported from a single patient simulator should be interpreted as conditional on the simulator and evaluation pipeline used to obtain it.
%U https://aclanthology.org/2026.aacl-long.266/
%P 5904-5923
Markdown (Informal)
[PULSE-CARE: A Measurement-Validity Audit of Simulated-Patient Clinical Agent Evaluation](https://aclanthology.org/2026.aacl-long.266/) (Dutta et al., AACL-IJCNLP 2026)
ACL
- Srimonti Dutta, Dinky Mishra, Aditi Singh, and Akshata Kishore Moharir. 2026. PULSE-CARE: A Measurement-Validity Audit of Simulated-Patient Clinical Agent Evaluation. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 5904–5923, Hengqin, China. Association for Computational Linguistics.