@inproceedings{sata-nakayama-2026-revisiting,
title = "Revisiting Co-Evolution for Multi-Turn Safety Alignment: Limitations and Alignment Tax Mitigation",
author = "Sata, Yoshiaki and
Nakayama, Kouta",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.231/",
pages = "4565--4578",
ISBN = "979-8-89176-444-6",
abstract = "To enhance the multi-turn defense capabilities of Large Language Models, recent studies focus on a ``co-evolution framework.'' This framework iteratively trains an attacker model to explore vulnerabilities and a target model to learn safety alignment. However, in our preliminary experiments, existing methods suffered from severe training instability. To address this and align with recent optimization methodologies, we introduce GRPO into the training loops of both models. We found that a single alignment cycle sufficiently established the target model{'}s defense, which in turn stalled the subsequent attacker training. Under the GRPO-based attacker-training pipeline and experimental setting examined in this study, this limitation rendered the iterative co-evolution process ineffective. Furthermore, to maintain model utility, we mix harmful goals with safe goals automatically rewritten from them. In our experiments, mixing safe goals improved utility but yielded higher attack success rates than training without them, revealing a safety-utility trade-off in our setting. Warning: This paper includes examples of harmful language."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sata-nakayama-2026-revisiting">
<titleInfo>
<title>Revisiting Co-Evolution for Multi-Turn Safety Alignment: Limitations and Alignment Tax Mitigation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yoshiaki</namePart>
<namePart type="family">Sata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kouta</namePart>
<namePart type="family">Nakayama</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>To enhance the multi-turn defense capabilities of Large Language Models, recent studies focus on a “co-evolution framework.” This framework iteratively trains an attacker model to explore vulnerabilities and a target model to learn safety alignment. However, in our preliminary experiments, existing methods suffered from severe training instability. To address this and align with recent optimization methodologies, we introduce GRPO into the training loops of both models. We found that a single alignment cycle sufficiently established the target model’s defense, which in turn stalled the subsequent attacker training. Under the GRPO-based attacker-training pipeline and experimental setting examined in this study, this limitation rendered the iterative co-evolution process ineffective. Furthermore, to maintain model utility, we mix harmful goals with safe goals automatically rewritten from them. In our experiments, mixing safe goals improved utility but yielded higher attack success rates than training without them, revealing a safety-utility trade-off in our setting. Warning: This paper includes examples of harmful language.</abstract>
<identifier type="citekey">sata-nakayama-2026-revisiting</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.231/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4565</start>
<end>4578</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Revisiting Co-Evolution for Multi-Turn Safety Alignment: Limitations and Alignment Tax Mitigation
%A Sata, Yoshiaki
%A Nakayama, Kouta
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F sata-nakayama-2026-revisiting
%X To enhance the multi-turn defense capabilities of Large Language Models, recent studies focus on a “co-evolution framework.” This framework iteratively trains an attacker model to explore vulnerabilities and a target model to learn safety alignment. However, in our preliminary experiments, existing methods suffered from severe training instability. To address this and align with recent optimization methodologies, we introduce GRPO into the training loops of both models. We found that a single alignment cycle sufficiently established the target model’s defense, which in turn stalled the subsequent attacker training. Under the GRPO-based attacker-training pipeline and experimental setting examined in this study, this limitation rendered the iterative co-evolution process ineffective. Furthermore, to maintain model utility, we mix harmful goals with safe goals automatically rewritten from them. In our experiments, mixing safe goals improved utility but yielded higher attack success rates than training without them, revealing a safety-utility trade-off in our setting. Warning: This paper includes examples of harmful language.
%U https://aclanthology.org/2026.findings-aacl.231/
%P 4565-4578
Markdown (Informal)
[Revisiting Co-Evolution for Multi-Turn Safety Alignment: Limitations and Alignment Tax Mitigation](https://aclanthology.org/2026.findings-aacl.231/) (Sata & Nakayama, Findings 2026)
ACL