@inproceedings{jiang-etal-2026-med,
title = "{M}ed-{GRADE}: Medical Grading and Rubric-based Assessment of Doctor-Patient Encounters",
author = "Jiang, Yidong and
Ding, Chao and
lu, Renjie and
Zhang, Rongzhao and
Jiang, Yankai and
Li, Tianbin and
Ning, Junzhi and
Li, Chengze and
Zhang, Haoyu and
Ding, Jinru and
Chen, Pengcheng",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.220/",
pages = "3673--3694",
ISBN = "979-8-89176-460-6",
abstract = "Large language model (LLM)-based judges are increasingly used to scale the evaluation and optimization of open-ended medical dialogue systems, making their reliability critical in this high-stakes setting. However, existing medical benchmarks mainly assess their medical knowledge, while general judge benchmarks lack clinically grounded consultation criteria. To address this gap, we introduce Med-GRADE, a benchmark for evaluating LLMs as medical dialogue judges. It contains 2,076 real-world patient{--}LLM dialogues annotated by 56 physicians with a 23-item rubric adapted from Mini-CEX, covering Medical Interviewing (MI), Humanistic Care (HC), and Diagnosis and Treatment Management (DTM). Experiments on 17 LLMs show that current LLM judges achieve only moderate agreement, with the best model reaching 64.89{\%} Macro-F1 and 68.28{\%} Hamming Accuracy. Besides, more complex prompts may even harm performance: CoT disrupts parsing, while few-shot and self-refine introduce item-level instability. Data and code are available at https://github.com/jyd777/Med-GRADE for reproducibility."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jiang-etal-2026-med">
<titleInfo>
<title>Med-GRADE: Medical Grading and Rubric-based Assessment of Doctor-Patient Encounters</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yidong</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Renjie</namePart>
<namePart type="family">lu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rongzhao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yankai</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianbin</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junzhi</namePart>
<namePart type="family">Ning</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chengze</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoyu</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jinru</namePart>
<namePart type="family">Ding</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pengcheng</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Large language model (LLM)-based judges are increasingly used to scale the evaluation and optimization of open-ended medical dialogue systems, making their reliability critical in this high-stakes setting. However, existing medical benchmarks mainly assess their medical knowledge, while general judge benchmarks lack clinically grounded consultation criteria. To address this gap, we introduce Med-GRADE, a benchmark for evaluating LLMs as medical dialogue judges. It contains 2,076 real-world patient–LLM dialogues annotated by 56 physicians with a 23-item rubric adapted from Mini-CEX, covering Medical Interviewing (MI), Humanistic Care (HC), and Diagnosis and Treatment Management (DTM). Experiments on 17 LLMs show that current LLM judges achieve only moderate agreement, with the best model reaching 64.89% Macro-F1 and 68.28% Hamming Accuracy. Besides, more complex prompts may even harm performance: CoT disrupts parsing, while few-shot and self-refine introduce item-level instability. Data and code are available at https://github.com/jyd777/Med-GRADE for reproducibility.</abstract>
<identifier type="citekey">jiang-etal-2026-med</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.220/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>3673</start>
<end>3694</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Med-GRADE: Medical Grading and Rubric-based Assessment of Doctor-Patient Encounters
%A Jiang, Yidong
%A Ding, Chao
%A lu, Renjie
%A Zhang, Rongzhao
%A Jiang, Yankai
%A Li, Tianbin
%A Ning, Junzhi
%A Li, Chengze
%A Zhang, Haoyu
%A Ding, Jinru
%A Chen, Pengcheng
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F jiang-etal-2026-med
%X Large language model (LLM)-based judges are increasingly used to scale the evaluation and optimization of open-ended medical dialogue systems, making their reliability critical in this high-stakes setting. However, existing medical benchmarks mainly assess their medical knowledge, while general judge benchmarks lack clinically grounded consultation criteria. To address this gap, we introduce Med-GRADE, a benchmark for evaluating LLMs as medical dialogue judges. It contains 2,076 real-world patient–LLM dialogues annotated by 56 physicians with a 23-item rubric adapted from Mini-CEX, covering Medical Interviewing (MI), Humanistic Care (HC), and Diagnosis and Treatment Management (DTM). Experiments on 17 LLMs show that current LLM judges achieve only moderate agreement, with the best model reaching 64.89% Macro-F1 and 68.28% Hamming Accuracy. Besides, more complex prompts may even harm performance: CoT disrupts parsing, while few-shot and self-refine introduce item-level instability. Data and code are available at https://github.com/jyd777/Med-GRADE for reproducibility.
%U https://aclanthology.org/2026.emnlp-industry.220/
%P 3673-3694
Markdown (Informal)
[Med-GRADE: Medical Grading and Rubric-based Assessment of Doctor-Patient Encounters](https://aclanthology.org/2026.emnlp-industry.220/) (Jiang et al., EMNLP 2026)
ACL
- Yidong Jiang, Chao Ding, Renjie lu, Rongzhao Zhang, Yankai Jiang, Tianbin Li, Junzhi Ning, Chengze Li, Haoyu Zhang, Jinru Ding, and Pengcheng Chen. 2026. Med-GRADE: Medical Grading and Rubric-based Assessment of Doctor-Patient Encounters. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 3673–3694, Budapest, Hungary. Association for Computational Linguistics.