@inproceedings{wang-etal-2026-cognitive-level,
title = "Cognitive-Level Prefix-{M}o{E} Reward Modeling for Curriculum Alignment of Medical Language Models",
author = "Wang, Hao and
Feng, Wendi and
Kang, Chun and
Wang, Yanting and
Li, Rui and
Sha, Lei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.611/",
pages = "14065--14079",
ISBN = "979-8-89176-459-0",
abstract = "Medical language models are increasingly evaluated on open-ended clinical reasoning, where good responses must combine factual recall, evidence integration, decision making, and synthesis. However, standard reward models compress these different standards into one reward score, while medical RL with answer keys mainly rewards multiple-choice correctness. This makes it difficult to improve non-multiple-choice medical reasoning without losing performance on exam-style benchmarks. In this paper, we propose a Prefix Mixture-of-Experts Reward Model (Prefix-MoE RM), which routes each query-response pair across recall, analysis, decision, and synthesis prefix experts, softly fuses their trainable prefixes, and then predicts one reward score. The MoE structure allows different abilities to use different scoring experts to reduce reward conflicts, while continuous prefixes provide a more expressive conditioning signal than fixed hard prompts. The reward model is trained on non-multiple-choice medical preferences and then used in mixed-data curriculum GRPO, where multiple-choice prompts provide additional answer-key feedback. On held-out reward-model evaluation, Prefix-MoE RM improves pairwise accuracy from 79.8 to 93.0 over a shared RM; with the RL pipeline fixed, replacing it with a shared RM or soft expert-head MoE lowers HealthBench from 43.8 to 24.8 and 34.6, respectively. With Qwen3-8B as the policy backbone, the full system outperforms the larger Qwen3-32B base on MedXpertQA and HealthBench, while the Qwen3-32B version reaches 50.1 HealthBench and 4.55 LLMEval-Med, remaining competitive with stronger closed general models and retaining an advantage on non-multiple-choice metrics."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-cognitive-level">
<titleInfo>
<title>Cognitive-Level Prefix-MoE Reward Modeling for Curriculum Alignment of Medical Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wendi</namePart>
<namePart type="family">Feng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chun</namePart>
<namePart type="family">Kang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yanting</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lei</namePart>
<namePart type="family">Sha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Medical language models are increasingly evaluated on open-ended clinical reasoning, where good responses must combine factual recall, evidence integration, decision making, and synthesis. However, standard reward models compress these different standards into one reward score, while medical RL with answer keys mainly rewards multiple-choice correctness. This makes it difficult to improve non-multiple-choice medical reasoning without losing performance on exam-style benchmarks. In this paper, we propose a Prefix Mixture-of-Experts Reward Model (Prefix-MoE RM), which routes each query-response pair across recall, analysis, decision, and synthesis prefix experts, softly fuses their trainable prefixes, and then predicts one reward score. The MoE structure allows different abilities to use different scoring experts to reduce reward conflicts, while continuous prefixes provide a more expressive conditioning signal than fixed hard prompts. The reward model is trained on non-multiple-choice medical preferences and then used in mixed-data curriculum GRPO, where multiple-choice prompts provide additional answer-key feedback. On held-out reward-model evaluation, Prefix-MoE RM improves pairwise accuracy from 79.8 to 93.0 over a shared RM; with the RL pipeline fixed, replacing it with a shared RM or soft expert-head MoE lowers HealthBench from 43.8 to 24.8 and 34.6, respectively. With Qwen3-8B as the policy backbone, the full system outperforms the larger Qwen3-32B base on MedXpertQA and HealthBench, while the Qwen3-32B version reaches 50.1 HealthBench and 4.55 LLMEval-Med, remaining competitive with stronger closed general models and retaining an advantage on non-multiple-choice metrics.</abstract>
<identifier type="citekey">wang-etal-2026-cognitive-level</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.611/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>14065</start>
<end>14079</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Cognitive-Level Prefix-MoE Reward Modeling for Curriculum Alignment of Medical Language Models
%A Wang, Hao
%A Feng, Wendi
%A Kang, Chun
%A Wang, Yanting
%A Li, Rui
%A Sha, Lei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wang-etal-2026-cognitive-level
%X Medical language models are increasingly evaluated on open-ended clinical reasoning, where good responses must combine factual recall, evidence integration, decision making, and synthesis. However, standard reward models compress these different standards into one reward score, while medical RL with answer keys mainly rewards multiple-choice correctness. This makes it difficult to improve non-multiple-choice medical reasoning without losing performance on exam-style benchmarks. In this paper, we propose a Prefix Mixture-of-Experts Reward Model (Prefix-MoE RM), which routes each query-response pair across recall, analysis, decision, and synthesis prefix experts, softly fuses their trainable prefixes, and then predicts one reward score. The MoE structure allows different abilities to use different scoring experts to reduce reward conflicts, while continuous prefixes provide a more expressive conditioning signal than fixed hard prompts. The reward model is trained on non-multiple-choice medical preferences and then used in mixed-data curriculum GRPO, where multiple-choice prompts provide additional answer-key feedback. On held-out reward-model evaluation, Prefix-MoE RM improves pairwise accuracy from 79.8 to 93.0 over a shared RM; with the RL pipeline fixed, replacing it with a shared RM or soft expert-head MoE lowers HealthBench from 43.8 to 24.8 and 34.6, respectively. With Qwen3-8B as the policy backbone, the full system outperforms the larger Qwen3-32B base on MedXpertQA and HealthBench, while the Qwen3-32B version reaches 50.1 HealthBench and 4.55 LLMEval-Med, remaining competitive with stronger closed general models and retaining an advantage on non-multiple-choice metrics.
%U https://aclanthology.org/2026.emnlp-main.611/
%P 14065-14079
Markdown (Informal)
[Cognitive-Level Prefix-MoE Reward Modeling for Curriculum Alignment of Medical Language Models](https://aclanthology.org/2026.emnlp-main.611/) (Wang et al., EMNLP 2026)
ACL