@inproceedings{anantaprayoon-etal-2026-learning,
title = "Learning to Negotiate: Multi-Agent Deliberation for Collective Value Alignment in {LLM}s",
author = "Anantaprayoon, Panatchakorn and
Babina, Nataliia and
Asgharbeygi, Nima and
Tarifi, Jad",
editor = "Anastasiou, Lucas and
Boland, Katarina and
Liddo, Anna De and
Falk, Neele and
Hautli-Janisz, Annette and
Lapesa, Gabriella and
Romberg, Julia",
booktitle = "Proceedings of The 2nd Workshop on Language-driven Deliberation Technology",
month = may,
year = "2026",
address = "Mallorca, Spain",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.delite-1.4/",
doi = "10.63317/3ochfxry9rdd",
pages = "29--48",
abstract = "LLM alignment has progressed in single-agent settings through paradigms such as RL with human feedback (RLHF), while recent work explores scalable alternatives such as RL with AI feedback (RLAIF) and dynamic alignment objectives. However, these approaches remain limited in multi-stakeholder settings, where conflicting values arise and deliberative negotiation is required. This work proposes a multi-agent negotiation-based alignment framework that aligns LLMs to Collective Agency (CA){---}an existing alignment objective introduced to promote the continual expansion of agency{---}while simultaneously improving conflict-resolution capability. To enable scalable training, two self-play LLM instances are assigned opposing personas and engage in turn-based dialogue to synthesize mutually beneficial solutions. We generate synthetic moral-dilemma prompts and conflicting persona pairs, and optimize the policy via RLAIF using Group Relative Policy Optimization (GRPO) with an external LLM reward model. While rewards are computed from CA scores assigned to the final completion, gradients are applied to dialogue tokens to directly improve deliberative interaction dynamics. Experiments show that the model achieves CA alignment comparable to a single-agent baseline while substantially improving conflict-resolution performance without degrading general language capabilities. These results suggest that negotiation-driven deliberation training provides a practical path toward LLMs that better support collective decision-making in value-conflict scenarios."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="anantaprayoon-etal-2026-learning">
<titleInfo>
<title>Learning to Negotiate: Multi-Agent Deliberation for Collective Value Alignment in LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Panatchakorn</namePart>
<namePart type="family">Anantaprayoon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nataliia</namePart>
<namePart type="family">Babina</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nima</namePart>
<namePart type="family">Asgharbeygi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jad</namePart>
<namePart type="family">Tarifi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of The 2nd Workshop on Language-driven Deliberation Technology</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lucas</namePart>
<namePart type="family">Anastasiou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Katarina</namePart>
<namePart type="family">Boland</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="given">De</namePart>
<namePart type="family">Liddo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neele</namePart>
<namePart type="family">Falk</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Annette</namePart>
<namePart type="family">Hautli-Janisz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriella</namePart>
<namePart type="family">Lapesa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julia</namePart>
<namePart type="family">Romberg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>LLM alignment has progressed in single-agent settings through paradigms such as RL with human feedback (RLHF), while recent work explores scalable alternatives such as RL with AI feedback (RLAIF) and dynamic alignment objectives. However, these approaches remain limited in multi-stakeholder settings, where conflicting values arise and deliberative negotiation is required. This work proposes a multi-agent negotiation-based alignment framework that aligns LLMs to Collective Agency (CA)—an existing alignment objective introduced to promote the continual expansion of agency—while simultaneously improving conflict-resolution capability. To enable scalable training, two self-play LLM instances are assigned opposing personas and engage in turn-based dialogue to synthesize mutually beneficial solutions. We generate synthetic moral-dilemma prompts and conflicting persona pairs, and optimize the policy via RLAIF using Group Relative Policy Optimization (GRPO) with an external LLM reward model. While rewards are computed from CA scores assigned to the final completion, gradients are applied to dialogue tokens to directly improve deliberative interaction dynamics. Experiments show that the model achieves CA alignment comparable to a single-agent baseline while substantially improving conflict-resolution performance without degrading general language capabilities. These results suggest that negotiation-driven deliberation training provides a practical path toward LLMs that better support collective decision-making in value-conflict scenarios.</abstract>
<identifier type="citekey">anantaprayoon-etal-2026-learning</identifier>
<identifier type="doi">10.63317/3ochfxry9rdd</identifier>
<location>
<url>https://aclanthology.org/2026.delite-1.4/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>29</start>
<end>48</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Learning to Negotiate: Multi-Agent Deliberation for Collective Value Alignment in LLMs
%A Anantaprayoon, Panatchakorn
%A Babina, Nataliia
%A Asgharbeygi, Nima
%A Tarifi, Jad
%Y Anastasiou, Lucas
%Y Boland, Katarina
%Y Liddo, Anna De
%Y Falk, Neele
%Y Hautli-Janisz, Annette
%Y Lapesa, Gabriella
%Y Romberg, Julia
%S Proceedings of The 2nd Workshop on Language-driven Deliberation Technology
%D 2026
%8 May
%I Association for Computational Linguistics
%C Mallorca, Spain
%F anantaprayoon-etal-2026-learning
%X LLM alignment has progressed in single-agent settings through paradigms such as RL with human feedback (RLHF), while recent work explores scalable alternatives such as RL with AI feedback (RLAIF) and dynamic alignment objectives. However, these approaches remain limited in multi-stakeholder settings, where conflicting values arise and deliberative negotiation is required. This work proposes a multi-agent negotiation-based alignment framework that aligns LLMs to Collective Agency (CA)—an existing alignment objective introduced to promote the continual expansion of agency—while simultaneously improving conflict-resolution capability. To enable scalable training, two self-play LLM instances are assigned opposing personas and engage in turn-based dialogue to synthesize mutually beneficial solutions. We generate synthetic moral-dilemma prompts and conflicting persona pairs, and optimize the policy via RLAIF using Group Relative Policy Optimization (GRPO) with an external LLM reward model. While rewards are computed from CA scores assigned to the final completion, gradients are applied to dialogue tokens to directly improve deliberative interaction dynamics. Experiments show that the model achieves CA alignment comparable to a single-agent baseline while substantially improving conflict-resolution performance without degrading general language capabilities. These results suggest that negotiation-driven deliberation training provides a practical path toward LLMs that better support collective decision-making in value-conflict scenarios.
%R 10.63317/3ochfxry9rdd
%U https://aclanthology.org/2026.delite-1.4/
%U https://doi.org/10.63317/3ochfxry9rdd
%P 29-48
Markdown (Informal)
[Learning to Negotiate: Multi-Agent Deliberation for Collective Value Alignment in LLMs](https://aclanthology.org/2026.delite-1.4/) (Anantaprayoon et al., DELITE 2026)
ACL