@inproceedings{joshi-etal-2026-many,
title = "Too Many Cooks Spoil The Broth: Auditing Multi-Agent {LLM} Councils For Medical Reasoning",
author = "Joshi, Mukul and
Jalan, Pratik and
Magotra, Akhilesh and
Jadhav, Kshitij Sharad",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2164/",
pages = "49230--49245",
ISBN = "979-8-89176-459-0",
abstract = "Multi-agent debate among large language models (LLMs) is widely assumed to improve reasoning by letting models correct one another. We audit that assumption for English medical question answering (QA). Five open-weight 4 to 8B medical and generalist LLMs answer 8{,}402 questions across six benchmarks as a council, measured against the same five prompted individually and a single larger Qwen3-VL-32B, text-only. The council beats every member on five of six datasets, but most of that accuracy is in place before debate runs: 67{\%} of verdicts settle at Round 0, and on five of six those are the more accurate questions. Where debate runs, opinion change concentrates on one anchor; on the hardest benchmark 85{\%} of moves toward it happen while it is wrong. There, agreement is worth less than disagreement: questions all five agree on are correct 6.6{\%} of the time against 14.6{\%} otherwise, tracking a shared bias toward the first option. The council also trails the larger model on four of six by up to 16.2 points, at seven times the model calls for 1.6 times the compute. Agreement among models is a measurement that can be biased, not evidence of correctness."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="joshi-etal-2026-many">
<titleInfo>
<title>Too Many Cooks Spoil The Broth: Auditing Multi-Agent LLM Councils For Medical Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mukul</namePart>
<namePart type="family">Joshi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pratik</namePart>
<namePart type="family">Jalan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Akhilesh</namePart>
<namePart type="family">Magotra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kshitij</namePart>
<namePart type="given">Sharad</namePart>
<namePart type="family">Jadhav</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Multi-agent debate among large language models (LLMs) is widely assumed to improve reasoning by letting models correct one another. We audit that assumption for English medical question answering (QA). Five open-weight 4 to 8B medical and generalist LLMs answer 8,402 questions across six benchmarks as a council, measured against the same five prompted individually and a single larger Qwen3-VL-32B, text-only. The council beats every member on five of six datasets, but most of that accuracy is in place before debate runs: 67% of verdicts settle at Round 0, and on five of six those are the more accurate questions. Where debate runs, opinion change concentrates on one anchor; on the hardest benchmark 85% of moves toward it happen while it is wrong. There, agreement is worth less than disagreement: questions all five agree on are correct 6.6% of the time against 14.6% otherwise, tracking a shared bias toward the first option. The council also trails the larger model on four of six by up to 16.2 points, at seven times the model calls for 1.6 times the compute. Agreement among models is a measurement that can be biased, not evidence of correctness.</abstract>
<identifier type="citekey">joshi-etal-2026-many</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2164/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>49230</start>
<end>49245</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Too Many Cooks Spoil The Broth: Auditing Multi-Agent LLM Councils For Medical Reasoning
%A Joshi, Mukul
%A Jalan, Pratik
%A Magotra, Akhilesh
%A Jadhav, Kshitij Sharad
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F joshi-etal-2026-many
%X Multi-agent debate among large language models (LLMs) is widely assumed to improve reasoning by letting models correct one another. We audit that assumption for English medical question answering (QA). Five open-weight 4 to 8B medical and generalist LLMs answer 8,402 questions across six benchmarks as a council, measured against the same five prompted individually and a single larger Qwen3-VL-32B, text-only. The council beats every member on five of six datasets, but most of that accuracy is in place before debate runs: 67% of verdicts settle at Round 0, and on five of six those are the more accurate questions. Where debate runs, opinion change concentrates on one anchor; on the hardest benchmark 85% of moves toward it happen while it is wrong. There, agreement is worth less than disagreement: questions all five agree on are correct 6.6% of the time against 14.6% otherwise, tracking a shared bias toward the first option. The council also trails the larger model on four of six by up to 16.2 points, at seven times the model calls for 1.6 times the compute. Agreement among models is a measurement that can be biased, not evidence of correctness.
%U https://aclanthology.org/2026.emnlp-main.2164/
%P 49230-49245
Markdown (Informal)
[Too Many Cooks Spoil The Broth: Auditing Multi-Agent LLM Councils For Medical Reasoning](https://aclanthology.org/2026.emnlp-main.2164/) (Joshi et al., EMNLP 2026)
ACL