@inproceedings{choi-etal-2026-text-capability,
title = "Text Capability Loss in Vision-Language Adaptation: An Attention-Sink Diagnosis",
author = "Choi, Minsik and
Kim, Geewook and
Kim, Young Geun",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2537/",
pages = "57684--57712",
ISBN = "979-8-89176-459-0",
abstract = "Fine-tuning a pretrained LLM into a vision-language model (VLM) can erode the backbone{'}s text capability, with the damage concentrated on tasks that require following exact output rules, such as instruction following, chain-of-thought reasoning graded on a strictly parsed final answer, and similar evaluations with strict graders. We trace this gap to *attention-sink corruption*: VL fine-tuning perturbs the early sink position that anchors a large fraction of attention probability, and how well the base LLM preserves its sink tracks how much of the affected capability survives adaptation. Building on this view, we introduce *Sink Strength*, a single scalar computed on the base LLM in a few seconds on a single GPU that predicts post-VL degradation without any VL training. It consistently tracks relative degradation across the six VLM{--}LLM pairs and multiple format-sensitive tasks. Complementing this diagnostic, we find that post-pretraining QK-norm injection fails to reproduce the protection of native QK-norm, while several off-the-shelf weight-merging settings fail to recover the lost capability after VL training. These negative results underscore the value of screening backbones with Sink Strength before VL training and narrow the intervention space toward head-selective training-time protection."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="choi-etal-2026-text-capability">
<titleInfo>
<title>Text Capability Loss in Vision-Language Adaptation: An Attention-Sink Diagnosis</title>
</titleInfo>
<name type="personal">
<namePart type="given">Minsik</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Geewook</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Young</namePart>
<namePart type="given">Geun</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Fine-tuning a pretrained LLM into a vision-language model (VLM) can erode the backbone’s text capability, with the damage concentrated on tasks that require following exact output rules, such as instruction following, chain-of-thought reasoning graded on a strictly parsed final answer, and similar evaluations with strict graders. We trace this gap to *attention-sink corruption*: VL fine-tuning perturbs the early sink position that anchors a large fraction of attention probability, and how well the base LLM preserves its sink tracks how much of the affected capability survives adaptation. Building on this view, we introduce *Sink Strength*, a single scalar computed on the base LLM in a few seconds on a single GPU that predicts post-VL degradation without any VL training. It consistently tracks relative degradation across the six VLM–LLM pairs and multiple format-sensitive tasks. Complementing this diagnostic, we find that post-pretraining QK-norm injection fails to reproduce the protection of native QK-norm, while several off-the-shelf weight-merging settings fail to recover the lost capability after VL training. These negative results underscore the value of screening backbones with Sink Strength before VL training and narrow the intervention space toward head-selective training-time protection.</abstract>
<identifier type="citekey">choi-etal-2026-text-capability</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2537/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>57684</start>
<end>57712</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Text Capability Loss in Vision-Language Adaptation: An Attention-Sink Diagnosis
%A Choi, Minsik
%A Kim, Geewook
%A Kim, Young Geun
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F choi-etal-2026-text-capability
%X Fine-tuning a pretrained LLM into a vision-language model (VLM) can erode the backbone’s text capability, with the damage concentrated on tasks that require following exact output rules, such as instruction following, chain-of-thought reasoning graded on a strictly parsed final answer, and similar evaluations with strict graders. We trace this gap to *attention-sink corruption*: VL fine-tuning perturbs the early sink position that anchors a large fraction of attention probability, and how well the base LLM preserves its sink tracks how much of the affected capability survives adaptation. Building on this view, we introduce *Sink Strength*, a single scalar computed on the base LLM in a few seconds on a single GPU that predicts post-VL degradation without any VL training. It consistently tracks relative degradation across the six VLM–LLM pairs and multiple format-sensitive tasks. Complementing this diagnostic, we find that post-pretraining QK-norm injection fails to reproduce the protection of native QK-norm, while several off-the-shelf weight-merging settings fail to recover the lost capability after VL training. These negative results underscore the value of screening backbones with Sink Strength before VL training and narrow the intervention space toward head-selective training-time protection.
%U https://aclanthology.org/2026.emnlp-main.2537/
%P 57684-57712
Markdown (Informal)
[Text Capability Loss in Vision-Language Adaptation: An Attention-Sink Diagnosis](https://aclanthology.org/2026.emnlp-main.2537/) (Choi et al., EMNLP 2026)
ACL