@inproceedings{hu-etal-2026-token,
title = "Token-level Adaptive Weighting via Implicit Rewards for Inference-time Multi-objective Alignment",
author = "Hu, Dou and
Wei, Lingwei and
Xu, Peiran and
Xiao, Hongjiang and
Hu, Songlin and
Zhang, Yuan",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1928/",
pages = "41082--41096",
ISBN = "979-8-89176-462-0",
abstract = "Aligning large language models (LLMs) with human preferences is challenging due to the inherent heterogeneity and multi-dimensionality of these preferences. Existing inference-time multi-objective alignment methods often rely on fixed weights to combine preference models. However, different tokens contribute unequally to preference objectives during decoding. Ignoring such token-level variation may cause the loss of preference information and lead to objective interference. To alleviate this, we utilize the implicit rewards of preference models to estimate importance differences between tokens, and propose a new training-free and reward-free framework TAW to dynamically balance multiple objectives for inference-time alignment. We design the relative entropy and marginal entropy estimators to quantify the informativeness and uncertainty of preference objectives. TAW with the estimator ensures that more informative or reliable preferences are emphasized, while weak or uncertain signals are suppressed. Experiments on two LLM alignment benchmarks show that TAW outperforms strong baselines without predefined weightings, while also achieves an overall optimal Pareto front when explicit preference weighting is applied."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hu-etal-2026-token">
<titleInfo>
<title>Token-level Adaptive Weighting via Implicit Rewards for Inference-time Multi-objective Alignment</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dou</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lingwei</namePart>
<namePart type="family">Wei</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peiran</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongjiang</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Songlin</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Aligning large language models (LLMs) with human preferences is challenging due to the inherent heterogeneity and multi-dimensionality of these preferences. Existing inference-time multi-objective alignment methods often rely on fixed weights to combine preference models. However, different tokens contribute unequally to preference objectives during decoding. Ignoring such token-level variation may cause the loss of preference information and lead to objective interference. To alleviate this, we utilize the implicit rewards of preference models to estimate importance differences between tokens, and propose a new training-free and reward-free framework TAW to dynamically balance multiple objectives for inference-time alignment. We design the relative entropy and marginal entropy estimators to quantify the informativeness and uncertainty of preference objectives. TAW with the estimator ensures that more informative or reliable preferences are emphasized, while weak or uncertain signals are suppressed. Experiments on two LLM alignment benchmarks show that TAW outperforms strong baselines without predefined weightings, while also achieves an overall optimal Pareto front when explicit preference weighting is applied.</abstract>
<identifier type="citekey">hu-etal-2026-token</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1928/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41082</start>
<end>41096</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Token-level Adaptive Weighting via Implicit Rewards for Inference-time Multi-objective Alignment
%A Hu, Dou
%A Wei, Lingwei
%A Xu, Peiran
%A Xiao, Hongjiang
%A Hu, Songlin
%A Zhang, Yuan
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F hu-etal-2026-token
%X Aligning large language models (LLMs) with human preferences is challenging due to the inherent heterogeneity and multi-dimensionality of these preferences. Existing inference-time multi-objective alignment methods often rely on fixed weights to combine preference models. However, different tokens contribute unequally to preference objectives during decoding. Ignoring such token-level variation may cause the loss of preference information and lead to objective interference. To alleviate this, we utilize the implicit rewards of preference models to estimate importance differences between tokens, and propose a new training-free and reward-free framework TAW to dynamically balance multiple objectives for inference-time alignment. We design the relative entropy and marginal entropy estimators to quantify the informativeness and uncertainty of preference objectives. TAW with the estimator ensures that more informative or reliable preferences are emphasized, while weak or uncertain signals are suppressed. Experiments on two LLM alignment benchmarks show that TAW outperforms strong baselines without predefined weightings, while also achieves an overall optimal Pareto front when explicit preference weighting is applied.
%U https://aclanthology.org/2026.findings-emnlp.1928/
%P 41082-41096
Markdown (Informal)
[Token-level Adaptive Weighting via Implicit Rewards for Inference-time Multi-objective Alignment](https://aclanthology.org/2026.findings-emnlp.1928/) (Hu et al., Findings 2026)
ACL