@inproceedings{sarzotti-baratella-2026-teaching,
title = "Teaching or Sharpening? An Exploration of the Potential of {DPO} Post-Training for Dialogue Games",
author = "Sarzotti, Marika and
Baratella, Simone",
editor = "Suglia, Alessandro and
Giulianelli, Mario and
Schlangen, David and
Bernardi, Raffaella and
Fern{\'a}ndez, Raquel and
Hakimov, Sherzod and
McCallum, Sabrina and
Mazzaccara, Davide and
Le, Dieu-Thu and
Lemon, Oliver and
Koller, Alexander",
booktitle = "The Proceedings of Improving Language Models through Learning from Dialogue Game Interaction",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.lmplayschool-1.14/",
pages = "160--169",
ISBN = "979-8-89176-467-5",
abstract = "Direct Preference Optimization (DPO) has emerged as a lightweight alternative to RLHF for post-training LLMs, yet it remains unclear whether it can teach novel skills or only sharpen existing ones. We investigate this question in the context of dialogue games, constructing multiple sets of on-policy preference pairs from the rollouts of supervised-fine-tuned Qwen3.5 models (2B and 9B) on the clembench benchmark, targeting rule adherence, strategic behaviour and excessive verbosity. Across all ablations, performance tracks a single variable, namely the distance of the chosen response from the policy{'}s own distribution. Pairs built from the model{'}s own winning moves raise the clemscore from 58.82 to 70.21, while pairs built around GPT-corrected moves land below the baseline at both scales. Transcript analysis points to a structural reason: bare preference pairs reduce conditional rules to unconditional preferences, so that DPO proves able to elicit successful behaviour already latent in the policy, but not to instill behaviour it does not yet exhibit."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sarzotti-baratella-2026-teaching">
<titleInfo>
<title>Teaching or Sharpening? An Exploration of the Potential of DPO Post-Training for Dialogue Games</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marika</namePart>
<namePart type="family">Sarzotti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simone</namePart>
<namePart type="family">Baratella</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of Improving Language Models through Learning from Dialogue Game Interaction</title>
</titleInfo>
<name type="personal">
<namePart type="given">Alessandro</namePart>
<namePart type="family">Suglia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mario</namePart>
<namePart type="family">Giulianelli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Schlangen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raffaella</namePart>
<namePart type="family">Bernardi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raquel</namePart>
<namePart type="family">Fernández</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sherzod</namePart>
<namePart type="family">Hakimov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sabrina</namePart>
<namePart type="family">McCallum</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Davide</namePart>
<namePart type="family">Mazzaccara</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dieu-Thu</namePart>
<namePart type="family">Le</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oliver</namePart>
<namePart type="family">Lemon</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexander</namePart>
<namePart type="family">Koller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-467-5</identifier>
</relatedItem>
<abstract>Direct Preference Optimization (DPO) has emerged as a lightweight alternative to RLHF for post-training LLMs, yet it remains unclear whether it can teach novel skills or only sharpen existing ones. We investigate this question in the context of dialogue games, constructing multiple sets of on-policy preference pairs from the rollouts of supervised-fine-tuned Qwen3.5 models (2B and 9B) on the clembench benchmark, targeting rule adherence, strategic behaviour and excessive verbosity. Across all ablations, performance tracks a single variable, namely the distance of the chosen response from the policy’s own distribution. Pairs built from the model’s own winning moves raise the clemscore from 58.82 to 70.21, while pairs built around GPT-corrected moves land below the baseline at both scales. Transcript analysis points to a structural reason: bare preference pairs reduce conditional rules to unconditional preferences, so that DPO proves able to elicit successful behaviour already latent in the policy, but not to instill behaviour it does not yet exhibit.</abstract>
<identifier type="citekey">sarzotti-baratella-2026-teaching</identifier>
<location>
<url>https://aclanthology.org/2026.lmplayschool-1.14/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>160</start>
<end>169</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Teaching or Sharpening? An Exploration of the Potential of DPO Post-Training for Dialogue Games
%A Sarzotti, Marika
%A Baratella, Simone
%Y Suglia, Alessandro
%Y Giulianelli, Mario
%Y Schlangen, David
%Y Bernardi, Raffaella
%Y Fernández, Raquel
%Y Hakimov, Sherzod
%Y McCallum, Sabrina
%Y Mazzaccara, Davide
%Y Le, Dieu-Thu
%Y Lemon, Oliver
%Y Koller, Alexander
%S The Proceedings of Improving Language Models through Learning from Dialogue Game Interaction
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-467-5
%F sarzotti-baratella-2026-teaching
%X Direct Preference Optimization (DPO) has emerged as a lightweight alternative to RLHF for post-training LLMs, yet it remains unclear whether it can teach novel skills or only sharpen existing ones. We investigate this question in the context of dialogue games, constructing multiple sets of on-policy preference pairs from the rollouts of supervised-fine-tuned Qwen3.5 models (2B and 9B) on the clembench benchmark, targeting rule adherence, strategic behaviour and excessive verbosity. Across all ablations, performance tracks a single variable, namely the distance of the chosen response from the policy’s own distribution. Pairs built from the model’s own winning moves raise the clemscore from 58.82 to 70.21, while pairs built around GPT-corrected moves land below the baseline at both scales. Transcript analysis points to a structural reason: bare preference pairs reduce conditional rules to unconditional preferences, so that DPO proves able to elicit successful behaviour already latent in the policy, but not to instill behaviour it does not yet exhibit.
%U https://aclanthology.org/2026.lmplayschool-1.14/
%P 160-169
Markdown (Informal)
[Teaching or Sharpening? An Exploration of the Potential of DPO Post-Training for Dialogue Games](https://aclanthology.org/2026.lmplayschool-1.14/) (Sarzotti & Baratella, LM Playschool 2026)
ACL