@inproceedings{malandri-etal-2026-addressing,
title = "Addressing the Concept Faithfulness Gap through Probe-Guided Sparse Autoencoders",
author = "Malandri, Lorenzo and
Mercorio, Fabio and
Serino, Antonio",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.550/",
pages = "11593--11611",
ISBN = "979-8-89176-462-0",
abstract = {Sparse Autoencoders (SAEs) decompose language model activations into sparse, interpretable features, but these features often struggle to predict task-relevant concepts that remain clearly encoded in the original dense activations. We call this mismatch a concept faithfulness gap: a faithful SAE should preserve concept-level representations, such as toxicity or sentiment, rather than producing features that look interpretable but lose predictive relevance. We introduce ProbeSAE, a probing-guided fine-tuning framework that addresses this gap by preserving task-relevant concept structure in a more monosemantic sparse decomposition, enabling more effective causal control over model behaviour without sacrificing interpretability. We evaluate ProbeSAE on two SAE architectures (TopK, JumpReLU), two domains (toxicity, sentiment), and multilingual out-of-distribution settings spanning six languages. Trained exclusively on English data, ProbeSAE: (i) produces more predictive, monosemantic, and interpretable sparse features than pre-trained or na{\"i}vely fine-tuned SAEs across all six evaluation languages; (ii) learns language-independent feature directions, as confirmed by cross-lingual causal ablation; (iii) passes SAEBench evaluation, confirming these gains do not arise from task/domain overfitting. Together, these results show that the concept faithfulness gap in SAEs can be addressed through probe-guided training, improving monosemanticity and causal effectiveness without sacrificing interpretability, making probe-guided SAEs a more reliable tool for domain-specific interpretability analysis.}
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="malandri-etal-2026-addressing">
<titleInfo>
<title>Addressing the Concept Faithfulness Gap through Probe-Guided Sparse Autoencoders</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lorenzo</namePart>
<namePart type="family">Malandri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabio</namePart>
<namePart type="family">Mercorio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Serino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Sparse Autoencoders (SAEs) decompose language model activations into sparse, interpretable features, but these features often struggle to predict task-relevant concepts that remain clearly encoded in the original dense activations. We call this mismatch a concept faithfulness gap: a faithful SAE should preserve concept-level representations, such as toxicity or sentiment, rather than producing features that look interpretable but lose predictive relevance. We introduce ProbeSAE, a probing-guided fine-tuning framework that addresses this gap by preserving task-relevant concept structure in a more monosemantic sparse decomposition, enabling more effective causal control over model behaviour without sacrificing interpretability. We evaluate ProbeSAE on two SAE architectures (TopK, JumpReLU), two domains (toxicity, sentiment), and multilingual out-of-distribution settings spanning six languages. Trained exclusively on English data, ProbeSAE: (i) produces more predictive, monosemantic, and interpretable sparse features than pre-trained or naïvely fine-tuned SAEs across all six evaluation languages; (ii) learns language-independent feature directions, as confirmed by cross-lingual causal ablation; (iii) passes SAEBench evaluation, confirming these gains do not arise from task/domain overfitting. Together, these results show that the concept faithfulness gap in SAEs can be addressed through probe-guided training, improving monosemanticity and causal effectiveness without sacrificing interpretability, making probe-guided SAEs a more reliable tool for domain-specific interpretability analysis.</abstract>
<identifier type="citekey">malandri-etal-2026-addressing</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.550/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>11593</start>
<end>11611</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Addressing the Concept Faithfulness Gap through Probe-Guided Sparse Autoencoders
%A Malandri, Lorenzo
%A Mercorio, Fabio
%A Serino, Antonio
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F malandri-etal-2026-addressing
%X Sparse Autoencoders (SAEs) decompose language model activations into sparse, interpretable features, but these features often struggle to predict task-relevant concepts that remain clearly encoded in the original dense activations. We call this mismatch a concept faithfulness gap: a faithful SAE should preserve concept-level representations, such as toxicity or sentiment, rather than producing features that look interpretable but lose predictive relevance. We introduce ProbeSAE, a probing-guided fine-tuning framework that addresses this gap by preserving task-relevant concept structure in a more monosemantic sparse decomposition, enabling more effective causal control over model behaviour without sacrificing interpretability. We evaluate ProbeSAE on two SAE architectures (TopK, JumpReLU), two domains (toxicity, sentiment), and multilingual out-of-distribution settings spanning six languages. Trained exclusively on English data, ProbeSAE: (i) produces more predictive, monosemantic, and interpretable sparse features than pre-trained or naïvely fine-tuned SAEs across all six evaluation languages; (ii) learns language-independent feature directions, as confirmed by cross-lingual causal ablation; (iii) passes SAEBench evaluation, confirming these gains do not arise from task/domain overfitting. Together, these results show that the concept faithfulness gap in SAEs can be addressed through probe-guided training, improving monosemanticity and causal effectiveness without sacrificing interpretability, making probe-guided SAEs a more reliable tool for domain-specific interpretability analysis.
%U https://aclanthology.org/2026.findings-emnlp.550/
%P 11593-11611
Markdown (Informal)
[Addressing the Concept Faithfulness Gap through Probe-Guided Sparse Autoencoders](https://aclanthology.org/2026.findings-emnlp.550/) (Malandri et al., Findings 2026)
ACL