@inproceedings{zhang-etal-2026-mitigating,
title = "Mitigating Exploration Bias in {RL} for Multi-Instruction Following",
author = "Zhang, Mian and
Yin, Yueqin and
He, Kaiyu and
Wu, Peilin and
Zhang, Xinlu and
Zhou, Mingyuan and
Chen, Zhiyu",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.786/",
pages = "16624--16633",
ISBN = "979-8-89176-462-0",
abstract = "RL has emerged as a powerful paradigm for enhancing the instruction following capabilities of LLMs. While existing training recipes achieve substantial gains, we find that they suffer from **exploration bias towards easy instructions** when the training data has multiple instructions in a prompt. This bias is caused by two main reasons: 1) **the policy model{'}s initial ability to satisfy hard instructions is too low** to trigger successful exploration during RL training, so the optimization is biased to easy instructions and 2) canonical RL training recipes typically employ a cumulative reward (the number of instructions fulfilled), **treating all the instructions equally**; the policy model is biased to fulfill the easy instructions to get the same amount of rewards. To address these, we first propose two metrics to measure the exploration bias of instruction following and then introduce a two-stage framework to alleviate it: 1) Behavioral Bootstrapping: a lightweight rejection sampling fine-tuning before RL to activate hard instructions and 2) Scarcity-Aware Rewards: a new RL reward function that assigns rewards to instructions based on their empirical scarcity. Experiments show that the proposed metrics are highly correlated with model performance and our methods unleash the potential of RL training: our best models outperform the baselines by a significant margin across three verifiable instruction following benchmarks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-mitigating">
<titleInfo>
<title>Mitigating Exploration Bias in RL for Multi-Instruction Following</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mian</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yueqin</namePart>
<namePart type="family">Yin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaiyu</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peilin</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinlu</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mingyuan</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyu</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>RL has emerged as a powerful paradigm for enhancing the instruction following capabilities of LLMs. While existing training recipes achieve substantial gains, we find that they suffer from **exploration bias towards easy instructions** when the training data has multiple instructions in a prompt. This bias is caused by two main reasons: 1) **the policy model’s initial ability to satisfy hard instructions is too low** to trigger successful exploration during RL training, so the optimization is biased to easy instructions and 2) canonical RL training recipes typically employ a cumulative reward (the number of instructions fulfilled), **treating all the instructions equally**; the policy model is biased to fulfill the easy instructions to get the same amount of rewards. To address these, we first propose two metrics to measure the exploration bias of instruction following and then introduce a two-stage framework to alleviate it: 1) Behavioral Bootstrapping: a lightweight rejection sampling fine-tuning before RL to activate hard instructions and 2) Scarcity-Aware Rewards: a new RL reward function that assigns rewards to instructions based on their empirical scarcity. Experiments show that the proposed metrics are highly correlated with model performance and our methods unleash the potential of RL training: our best models outperform the baselines by a significant margin across three verifiable instruction following benchmarks.</abstract>
<identifier type="citekey">zhang-etal-2026-mitigating</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.786/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>16624</start>
<end>16633</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Mitigating Exploration Bias in RL for Multi-Instruction Following
%A Zhang, Mian
%A Yin, Yueqin
%A He, Kaiyu
%A Wu, Peilin
%A Zhang, Xinlu
%A Zhou, Mingyuan
%A Chen, Zhiyu
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhang-etal-2026-mitigating
%X RL has emerged as a powerful paradigm for enhancing the instruction following capabilities of LLMs. While existing training recipes achieve substantial gains, we find that they suffer from **exploration bias towards easy instructions** when the training data has multiple instructions in a prompt. This bias is caused by two main reasons: 1) **the policy model’s initial ability to satisfy hard instructions is too low** to trigger successful exploration during RL training, so the optimization is biased to easy instructions and 2) canonical RL training recipes typically employ a cumulative reward (the number of instructions fulfilled), **treating all the instructions equally**; the policy model is biased to fulfill the easy instructions to get the same amount of rewards. To address these, we first propose two metrics to measure the exploration bias of instruction following and then introduce a two-stage framework to alleviate it: 1) Behavioral Bootstrapping: a lightweight rejection sampling fine-tuning before RL to activate hard instructions and 2) Scarcity-Aware Rewards: a new RL reward function that assigns rewards to instructions based on their empirical scarcity. Experiments show that the proposed metrics are highly correlated with model performance and our methods unleash the potential of RL training: our best models outperform the baselines by a significant margin across three verifiable instruction following benchmarks.
%U https://aclanthology.org/2026.findings-emnlp.786/
%P 16624-16633
Markdown (Informal)
[Mitigating Exploration Bias in RL for Multi-Instruction Following](https://aclanthology.org/2026.findings-emnlp.786/) (Zhang et al., Findings 2026)
ACL