@inproceedings{xu-etal-2026-riemannian,
title = "{R}iemannian Large Language Modeling",
author = "XU, Shu and
Liu, Jiahong and
Zhuang, Wenhao and
li, Jiabao and
Wang, Yefan and
Sun, Ming and
Yang, Menglin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2087/",
pages = "44427--44440",
ISBN = "979-8-89176-462-0",
abstract = "Current large language models perform next-token prediction under the implicit assumption that all tokens lie in a shared Euclidean embedding space, which treats every token uniformly. However, natural language contains rich structural relationships, including hierarchical, compositional, and cyclic patterns, which the flat geometry of Euclidean space cannot model well. To address this limitation, we propose rLLM (Riemannian Large Language Modeling), which introduces a token-adaptive positive-definite diagonal metric tensor into the LM head. This metric dynamically adjusts the local quadratic score for each hidden state and its context, allowing the model to better capture the relationships among tokens and prompts, with less than 0.01{\%} additional parameters. We train rLLM from scratch across three different scales (2.8B to 5.0B parameters), covering the full pretraining, SFT, and RL pipeline. Extensive experiments show that rLLM improves Base CORE, a composite of 22 in-context learning tasks, by up to $+2.5$pp at the two smaller GPT scales and obtains higher ChatCORE at all three scales, with gains up to $+3.0$pp after RL. These results suggest that revisiting the geometric assumptions underlying token representations is a promising direction for building more expressive and capable language models. Code is available at https://github.com/digailab/riemannianllm."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="xu-etal-2026-riemannian">
<titleInfo>
<title>Riemannian Large Language Modeling</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shu</namePart>
<namePart type="family">XU</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiahong</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenhao</namePart>
<namePart type="family">Zhuang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiabao</namePart>
<namePart type="family">li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yefan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ming</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Menglin</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Current large language models perform next-token prediction under the implicit assumption that all tokens lie in a shared Euclidean embedding space, which treats every token uniformly. However, natural language contains rich structural relationships, including hierarchical, compositional, and cyclic patterns, which the flat geometry of Euclidean space cannot model well. To address this limitation, we propose rLLM (Riemannian Large Language Modeling), which introduces a token-adaptive positive-definite diagonal metric tensor into the LM head. This metric dynamically adjusts the local quadratic score for each hidden state and its context, allowing the model to better capture the relationships among tokens and prompts, with less than 0.01% additional parameters. We train rLLM from scratch across three different scales (2.8B to 5.0B parameters), covering the full pretraining, SFT, and RL pipeline. Extensive experiments show that rLLM improves Base CORE, a composite of 22 in-context learning tasks, by up to +2.5pp at the two smaller GPT scales and obtains higher ChatCORE at all three scales, with gains up to +3.0pp after RL. These results suggest that revisiting the geometric assumptions underlying token representations is a promising direction for building more expressive and capable language models. Code is available at https://github.com/digailab/riemannianllm.</abstract>
<identifier type="citekey">xu-etal-2026-riemannian</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2087/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44427</start>
<end>44440</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Riemannian Large Language Modeling
%A XU, Shu
%A Liu, Jiahong
%A Zhuang, Wenhao
%A li, Jiabao
%A Wang, Yefan
%A Sun, Ming
%A Yang, Menglin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F xu-etal-2026-riemannian
%X Current large language models perform next-token prediction under the implicit assumption that all tokens lie in a shared Euclidean embedding space, which treats every token uniformly. However, natural language contains rich structural relationships, including hierarchical, compositional, and cyclic patterns, which the flat geometry of Euclidean space cannot model well. To address this limitation, we propose rLLM (Riemannian Large Language Modeling), which introduces a token-adaptive positive-definite diagonal metric tensor into the LM head. This metric dynamically adjusts the local quadratic score for each hidden state and its context, allowing the model to better capture the relationships among tokens and prompts, with less than 0.01% additional parameters. We train rLLM from scratch across three different scales (2.8B to 5.0B parameters), covering the full pretraining, SFT, and RL pipeline. Extensive experiments show that rLLM improves Base CORE, a composite of 22 in-context learning tasks, by up to +2.5pp at the two smaller GPT scales and obtains higher ChatCORE at all three scales, with gains up to +3.0pp after RL. These results suggest that revisiting the geometric assumptions underlying token representations is a promising direction for building more expressive and capable language models. Code is available at https://github.com/digailab/riemannianllm.
%U https://aclanthology.org/2026.findings-emnlp.2087/
%P 44427-44440
Markdown (Informal)
[Riemannian Large Language Modeling](https://aclanthology.org/2026.findings-emnlp.2087/) (XU et al., Findings 2026)
ACL
- Shu XU, Jiahong Liu, Wenhao Zhuang, Jiabao li, Yefan Wang, Ming Sun, and Menglin Yang. 2026. Riemannian Large Language Modeling. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 44427–44440, Budapest, Hungary. Association for Computational Linguistics.