@inproceedings{kasnavieh-etal-2026-introlm,
title = "{I}ntro{LM}: Introspective Language Models via Prefilling-Time Self-Evaluation",
author = "Kasnavieh, Hossein Hosseini and
Haffari, Gholamreza and
Leckie, Christopher and
Toosi, Adel N.",
editor = "Liakata, Maria and
Moreira, Viviane P. and
Zhang, Jiajun and
Jurgens, David",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026",
month = jul,
year = "2026",
address = "San Diego, California, United States",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-acl.598/",
doi = "10.18653/v1/2026.findings-acl.598",
pages = "12313--12326",
ISBN = "979-8-89176-395-1",
abstract = "A major challenge for the operation of large language models (LLMs) is how to predict whether a specific LLM will produce sufficiently high-quality output for a given query. Existing approaches rely on external classifiers, most commonly BERT-based models, which suffer from limited context windows, constrained representational capacity, and additional computational overhead. We propose IntroLM, a method that enables causal language models to predict their own output quality during the prefilling phase without affecting generation using [CPX] tokens. By introducing token-conditional LoRA that activates only for the introspective [CPX] token, the model learns to predict the output quality for a given query while preserving the original backbone behavior and avoiding external evaluators. On question-answering benchmarks, IntroLM applied to Qwen3-8B achieves a ROC{--}AUC of 90{\%} for success prediction, outperforming a DeBERTa-v3-Large classifier by 14{\%}. When integrated into multi-model routing systems, IntroLM achieves superior cost{--}performance trade-offs, reducing end-to-end latency by up to 33{\%} and large-model usage by up to 50{\%} at matched reliability. Our code is available at https://github.com/hhosseini1377/LLM{\_}routing."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kasnavieh-etal-2026-introlm">
<titleInfo>
<title>IntroLM: Introspective Language Models via Prefilling-Time Self-Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hossein</namePart>
<namePart type="given">Hosseini</namePart>
<namePart type="family">Kasnavieh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christopher</namePart>
<namePart type="family">Leckie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adel</namePart>
<namePart type="given">N</namePart>
<namePart type="family">Toosi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: ACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Liakata</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viviane</namePart>
<namePart type="given">P</namePart>
<namePart type="family">Moreira</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiajun</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Jurgens</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, California, United States</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-395-1</identifier>
</relatedItem>
<abstract>A major challenge for the operation of large language models (LLMs) is how to predict whether a specific LLM will produce sufficiently high-quality output for a given query. Existing approaches rely on external classifiers, most commonly BERT-based models, which suffer from limited context windows, constrained representational capacity, and additional computational overhead. We propose IntroLM, a method that enables causal language models to predict their own output quality during the prefilling phase without affecting generation using [CPX] tokens. By introducing token-conditional LoRA that activates only for the introspective [CPX] token, the model learns to predict the output quality for a given query while preserving the original backbone behavior and avoiding external evaluators. On question-answering benchmarks, IntroLM applied to Qwen3-8B achieves a ROC–AUC of 90% for success prediction, outperforming a DeBERTa-v3-Large classifier by 14%. When integrated into multi-model routing systems, IntroLM achieves superior cost–performance trade-offs, reducing end-to-end latency by up to 33% and large-model usage by up to 50% at matched reliability. Our code is available at https://github.com/hhosseini1377/LLM_routing.</abstract>
<identifier type="citekey">kasnavieh-etal-2026-introlm</identifier>
<identifier type="doi">10.18653/v1/2026.findings-acl.598</identifier>
<location>
<url>https://aclanthology.org/2026.findings-acl.598/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>12313</start>
<end>12326</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T IntroLM: Introspective Language Models via Prefilling-Time Self-Evaluation
%A Kasnavieh, Hossein Hosseini
%A Haffari, Gholamreza
%A Leckie, Christopher
%A Toosi, Adel N.
%Y Liakata, Maria
%Y Moreira, Viviane P.
%Y Zhang, Jiajun
%Y Jurgens, David
%S Findings of the Association for Computational Linguistics: ACL 2026
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, California, United States
%@ 979-8-89176-395-1
%F kasnavieh-etal-2026-introlm
%X A major challenge for the operation of large language models (LLMs) is how to predict whether a specific LLM will produce sufficiently high-quality output for a given query. Existing approaches rely on external classifiers, most commonly BERT-based models, which suffer from limited context windows, constrained representational capacity, and additional computational overhead. We propose IntroLM, a method that enables causal language models to predict their own output quality during the prefilling phase without affecting generation using [CPX] tokens. By introducing token-conditional LoRA that activates only for the introspective [CPX] token, the model learns to predict the output quality for a given query while preserving the original backbone behavior and avoiding external evaluators. On question-answering benchmarks, IntroLM applied to Qwen3-8B achieves a ROC–AUC of 90% for success prediction, outperforming a DeBERTa-v3-Large classifier by 14%. When integrated into multi-model routing systems, IntroLM achieves superior cost–performance trade-offs, reducing end-to-end latency by up to 33% and large-model usage by up to 50% at matched reliability. Our code is available at https://github.com/hhosseini1377/LLM_routing.
%R 10.18653/v1/2026.findings-acl.598
%U https://aclanthology.org/2026.findings-acl.598/
%U https://doi.org/10.18653/v1/2026.findings-acl.598
%P 12313-12326
Markdown (Informal)
[IntroLM: Introspective Language Models via Prefilling-Time Self-Evaluation](https://aclanthology.org/2026.findings-acl.598/) (Kasnavieh et al., Findings 2026)
ACL