@inproceedings{s-i-etal-2026-exploring,
title = "Exploring Collaboration between a language and a non-language agent",
author = "S I, Harini and
Singh, Somesh Kumar and
Singla, Yaman Kumar and
Shah, Rajiv Ratn and
Doermann, David and
Krishnamurthy, Balaji",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2189/",
pages = "49764--49799",
ISBN = "979-8-89176-459-0",
abstract = "LLMs are increasingly deployed as orchestrators that coordinate specialized subagents to solve complex tasks. In many important domains (game playing, robotics, autonomous driving), the strongest available agents are not language models: their expertise is encoded in latent representations of policy, value, and learned features, not in words. Integrating such agents into LLM reasoning pipelines currently requires \textit{verbalization}: compressing rich continuous representations into sparse textual summaries at each interaction step. To study whether verbalization constitutes a structural bottleneck, we introduce \textsc{LLAMIA-Bench}, a suite of six collaborative chess tasks spanning three facets: behavioral imitation, state assessment, and natural-language explanation. Tasks are drawn from prominent problems in chess literature and industry, each unsolvable by either component alone, and selected to vary in how much relevant signal survives verbalization.\par Our experiments reveal a consistent \textit{verbalization debt}: the performance gap between verbalized and internalized integration widens throughout training and persists as the LLM scales from 4B to 14B parameters. The effect is sharpest on tasks whose target signal has no faithful text representation: on puzzle interest prediction, verbalized systems of all sizes remain near-chance ($\leq 12/100$) while internalization reaches $52/100$. To close this gap, we introduce \textit{latent state internalization}, which projects the subagent{'}s continuous representations directly into the LLM{'}s token stream as learned state tokens, with dynamic re-encoding as actions advance the environment state. A single 14B model, \textsc{LLAMIA}, trained under this paradigm matches or exceeds dedicated task specialists and frontier models including GPT-5.1 with tool access across all benchmark tasks, and generalizes out-of-distribution where task-specific finetunes collapse. In a human study with skilled chess players, LLAMIA passes as human in 61{\%} of gameplay trials and its commentary is preferred in 72.2{\%} of pairings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="s-i-etal-2026-exploring">
<titleInfo>
<title>Exploring Collaboration between a language and a non-language agent</title>
</titleInfo>
<name type="personal">
<namePart type="given">Harini</namePart>
<namePart type="family">S I</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Somesh</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yaman</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Singla</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rajiv</namePart>
<namePart type="given">Ratn</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">Doermann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Balaji</namePart>
<namePart type="family">Krishnamurthy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>LLMs are increasingly deployed as orchestrators that coordinate specialized subagents to solve complex tasks. In many important domains (game playing, robotics, autonomous driving), the strongest available agents are not language models: their expertise is encoded in latent representations of policy, value, and learned features, not in words. Integrating such agents into LLM reasoning pipelines currently requires verbalization: compressing rich continuous representations into sparse textual summaries at each interaction step. To study whether verbalization constitutes a structural bottleneck, we introduce LLAMIA-Bench, a suite of six collaborative chess tasks spanning three facets: behavioral imitation, state assessment, and natural-language explanation. Tasks are drawn from prominent problems in chess literature and industry, each unsolvable by either component alone, and selected to vary in how much relevant signal survives verbalization.\par Our experiments reveal a consistent verbalization debt: the performance gap between verbalized and internalized integration widens throughout training and persists as the LLM scales from 4B to 14B parameters. The effect is sharpest on tasks whose target signal has no faithful text representation: on puzzle interest prediction, verbalized systems of all sizes remain near-chance (łeq 12/100) while internalization reaches 52/100. To close this gap, we introduce latent state internalization, which projects the subagent’s continuous representations directly into the LLM’s token stream as learned state tokens, with dynamic re-encoding as actions advance the environment state. A single 14B model, LLAMIA, trained under this paradigm matches or exceeds dedicated task specialists and frontier models including GPT-5.1 with tool access across all benchmark tasks, and generalizes out-of-distribution where task-specific finetunes collapse. In a human study with skilled chess players, LLAMIA passes as human in 61% of gameplay trials and its commentary is preferred in 72.2% of pairings.</abstract>
<identifier type="citekey">s-i-etal-2026-exploring</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2189/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>49764</start>
<end>49799</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Exploring Collaboration between a language and a non-language agent
%A S I, Harini
%A Singh, Somesh Kumar
%A Singla, Yaman Kumar
%A Shah, Rajiv Ratn
%A Doermann, David
%A Krishnamurthy, Balaji
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F s-i-etal-2026-exploring
%X LLMs are increasingly deployed as orchestrators that coordinate specialized subagents to solve complex tasks. In many important domains (game playing, robotics, autonomous driving), the strongest available agents are not language models: their expertise is encoded in latent representations of policy, value, and learned features, not in words. Integrating such agents into LLM reasoning pipelines currently requires verbalization: compressing rich continuous representations into sparse textual summaries at each interaction step. To study whether verbalization constitutes a structural bottleneck, we introduce LLAMIA-Bench, a suite of six collaborative chess tasks spanning three facets: behavioral imitation, state assessment, and natural-language explanation. Tasks are drawn from prominent problems in chess literature and industry, each unsolvable by either component alone, and selected to vary in how much relevant signal survives verbalization.\par Our experiments reveal a consistent verbalization debt: the performance gap between verbalized and internalized integration widens throughout training and persists as the LLM scales from 4B to 14B parameters. The effect is sharpest on tasks whose target signal has no faithful text representation: on puzzle interest prediction, verbalized systems of all sizes remain near-chance (łeq 12/100) while internalization reaches 52/100. To close this gap, we introduce latent state internalization, which projects the subagent’s continuous representations directly into the LLM’s token stream as learned state tokens, with dynamic re-encoding as actions advance the environment state. A single 14B model, LLAMIA, trained under this paradigm matches or exceeds dedicated task specialists and frontier models including GPT-5.1 with tool access across all benchmark tasks, and generalizes out-of-distribution where task-specific finetunes collapse. In a human study with skilled chess players, LLAMIA passes as human in 61% of gameplay trials and its commentary is preferred in 72.2% of pairings.
%U https://aclanthology.org/2026.emnlp-main.2189/
%P 49764-49799
Markdown (Informal)
[Exploring Collaboration between a language and a non-language agent](https://aclanthology.org/2026.emnlp-main.2189/) (S I et al., EMNLP 2026)
ACL
- Harini S I, Somesh Kumar Singh, Yaman Kumar Singla, Rajiv Ratn Shah, David Doermann, and Balaji Krishnamurthy. 2026. Exploring Collaboration between a language and a non-language agent. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 49764–49799, Budapest, Hungary. Association for Computational Linguistics.