@inproceedings{albayda-2026-icl,
title = "Do {ICL} and Finetuned Misalignment Share a Causal Component? Evidence from Risky Financial Advice",
author = "Albayda, Albrave Raphael",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.45/",
pages = "783--794",
ISBN = "979-8-89176-476-7",
abstract = "Misalignment in language models can be induced through either in-context learning (ICL) or supervised finetuning (SFT), but it is uncertain whether they share an internal causal representation. We study this in Llama-3.1-8B-Instruct by comparing two setups built from a dataset promoting financial recklessness: the base model whose context window carries 1,024 examples, and a model organism finetuned on that dataset. We extract difference-in-means directions that contrast misaligned and benign responses to finance advice prompts. We make four observations. First, the two directions are aligned across depth, with cosine similarity 0.59 at layer 24. Second, the component transfers causally across induction routes: ablating the ICL direction from the SFT model lowers misalignment from 58.8{\%} to 47.4{\%}, ablating the SFT direction from the ICL model lowers it from 65.9{\%} to 46.3{\%}, and steering the ICL direction into the non-misaligned model raises it to 88.9{\%}. Third, the component only partially represents the behavior: in the ICL model, the ICL direction is still causally much stronger than the SFT direction, and substantial misalignment remains even under repeated ablation. Finally, the component is domain-specific: finance-to-medical interventions do not transfer. On net the evidence supports a partial, domain-specific, and causal ICL{--}SFT bridge."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="albayda-2026-icl">
<titleInfo>
<title>Do ICL and Finetuned Misalignment Share a Causal Component? Evidence from Risky Financial Advice</title>
</titleInfo>
<name type="personal">
<namePart type="given">Albrave</namePart>
<namePart type="given">Raphael</namePart>
<namePart type="family">Albayda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>Misalignment in language models can be induced through either in-context learning (ICL) or supervised finetuning (SFT), but it is uncertain whether they share an internal causal representation. We study this in Llama-3.1-8B-Instruct by comparing two setups built from a dataset promoting financial recklessness: the base model whose context window carries 1,024 examples, and a model organism finetuned on that dataset. We extract difference-in-means directions that contrast misaligned and benign responses to finance advice prompts. We make four observations. First, the two directions are aligned across depth, with cosine similarity 0.59 at layer 24. Second, the component transfers causally across induction routes: ablating the ICL direction from the SFT model lowers misalignment from 58.8% to 47.4%, ablating the SFT direction from the ICL model lowers it from 65.9% to 46.3%, and steering the ICL direction into the non-misaligned model raises it to 88.9%. Third, the component only partially represents the behavior: in the ICL model, the ICL direction is still causally much stronger than the SFT direction, and substantial misalignment remains even under repeated ablation. Finally, the component is domain-specific: finance-to-medical interventions do not transfer. On net the evidence supports a partial, domain-specific, and causal ICL–SFT bridge.</abstract>
<identifier type="citekey">albayda-2026-icl</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.45/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>783</start>
<end>794</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Do ICL and Finetuned Misalignment Share a Causal Component? Evidence from Risky Financial Advice
%A Albayda, Albrave Raphael
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F albayda-2026-icl
%X Misalignment in language models can be induced through either in-context learning (ICL) or supervised finetuning (SFT), but it is uncertain whether they share an internal causal representation. We study this in Llama-3.1-8B-Instruct by comparing two setups built from a dataset promoting financial recklessness: the base model whose context window carries 1,024 examples, and a model organism finetuned on that dataset. We extract difference-in-means directions that contrast misaligned and benign responses to finance advice prompts. We make four observations. First, the two directions are aligned across depth, with cosine similarity 0.59 at layer 24. Second, the component transfers causally across induction routes: ablating the ICL direction from the SFT model lowers misalignment from 58.8% to 47.4%, ablating the SFT direction from the ICL model lowers it from 65.9% to 46.3%, and steering the ICL direction into the non-misaligned model raises it to 88.9%. Third, the component only partially represents the behavior: in the ICL model, the ICL direction is still causally much stronger than the SFT direction, and substantial misalignment remains even under repeated ablation. Finally, the component is domain-specific: finance-to-medical interventions do not transfer. On net the evidence supports a partial, domain-specific, and causal ICL–SFT bridge.
%U https://aclanthology.org/2026.blackboxnlp-1.45/
%P 783-794
Markdown (Informal)
[Do ICL and Finetuned Misalignment Share a Causal Component? Evidence from Risky Financial Advice](https://aclanthology.org/2026.blackboxnlp-1.45/) (Albayda, BlackboxNLP 2026)
ACL