@inproceedings{xue-etal-2026-beyond-analysis,
title = "Beyond Analysis: Training Language Models with Internal Mechanistic Feedback",
author = "Xue, Hanlin and
Zhang, Gang and
Wang, Xiangyuan and
Bai, Yunhao and
Wan, Yulong and
Li, Xiang and
Zhou, Jiaqi and
Xiong, Ruochong and
Liu, Junfei",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2604/",
pages = "59164--59178",
ISBN = "979-8-89176-459-0",
abstract = "Recent advances in mechanistic interpretability have revealed how language models process information, yet these insights rarely improve model performance. We propose Interpretable Feature-Space Regularization (IFSR), a training-time framework that transforms mechanistic insights into optimization signals. IFSR identifies error-prone feature interactions through circuit attribution and penalizes them during training, encouraging the model to discover alternative computational pathways. Unlike inference-time interventions that operate on individual features, IFSR targets feature-to-feature edges representing internal computational patterns, and permanently encodes improvements into model parameters. Experiments across ten classification tasks show consistent improvements. Cross-task evaluation demonstrates that IFSR training can transfer positively to unrelated tasks, suggesting benefits to general model capabilities. Our analysis reveals that most identified error patterns resist human interpretation, yet penalizing them still improves performance, suggesting that automatic error identification at the level of feature interactions is feasible and effective. This work demonstrates that mechanistic interpretability can directly enhance task performance through training-time optimization."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xue-etal-2026-beyond-analysis">
<titleInfo>
<title>Beyond Analysis: Training Language Models with Internal Mechanistic Feedback</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hanlin</namePart>
<namePart type="family">Xue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gang</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiangyuan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yunhao</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yulong</namePart>
<namePart type="family">Wan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiaqi</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruochong</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junfei</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Recent advances in mechanistic interpretability have revealed how language models process information, yet these insights rarely improve model performance. We propose Interpretable Feature-Space Regularization (IFSR), a training-time framework that transforms mechanistic insights into optimization signals. IFSR identifies error-prone feature interactions through circuit attribution and penalizes them during training, encouraging the model to discover alternative computational pathways. Unlike inference-time interventions that operate on individual features, IFSR targets feature-to-feature edges representing internal computational patterns, and permanently encodes improvements into model parameters. Experiments across ten classification tasks show consistent improvements. Cross-task evaluation demonstrates that IFSR training can transfer positively to unrelated tasks, suggesting benefits to general model capabilities. Our analysis reveals that most identified error patterns resist human interpretation, yet penalizing them still improves performance, suggesting that automatic error identification at the level of feature interactions is feasible and effective. This work demonstrates that mechanistic interpretability can directly enhance task performance through training-time optimization.</abstract>
<identifier type="citekey">xue-etal-2026-beyond-analysis</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2604/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>59164</start>
<end>59178</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Analysis: Training Language Models with Internal Mechanistic Feedback
%A Xue, Hanlin
%A Zhang, Gang
%A Wang, Xiangyuan
%A Bai, Yunhao
%A Wan, Yulong
%A Li, Xiang
%A Zhou, Jiaqi
%A Xiong, Ruochong
%A Liu, Junfei
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F xue-etal-2026-beyond-analysis
%X Recent advances in mechanistic interpretability have revealed how language models process information, yet these insights rarely improve model performance. We propose Interpretable Feature-Space Regularization (IFSR), a training-time framework that transforms mechanistic insights into optimization signals. IFSR identifies error-prone feature interactions through circuit attribution and penalizes them during training, encouraging the model to discover alternative computational pathways. Unlike inference-time interventions that operate on individual features, IFSR targets feature-to-feature edges representing internal computational patterns, and permanently encodes improvements into model parameters. Experiments across ten classification tasks show consistent improvements. Cross-task evaluation demonstrates that IFSR training can transfer positively to unrelated tasks, suggesting benefits to general model capabilities. Our analysis reveals that most identified error patterns resist human interpretation, yet penalizing them still improves performance, suggesting that automatic error identification at the level of feature interactions is feasible and effective. This work demonstrates that mechanistic interpretability can directly enhance task performance through training-time optimization.
%U https://aclanthology.org/2026.emnlp-main.2604/
%P 59164-59178
Markdown (Informal)
[Beyond Analysis: Training Language Models with Internal Mechanistic Feedback](https://aclanthology.org/2026.emnlp-main.2604/) (Xue et al., EMNLP 2026)
ACL
- Hanlin Xue, Gang Zhang, Xiangyuan Wang, Yunhao Bai, Yulong Wan, Xiang Li, Jiaqi Zhou, Ruochong Xiong, and Junfei Liu. 2026. Beyond Analysis: Training Language Models with Internal Mechanistic Feedback. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 59164–59178, Budapest, Hungary. Association for Computational Linguistics.