@inproceedings{ye-etal-2026-ttml,
title = "{TTML}: Test-Time Memory Learning for Label-Free {RLVR}",
author = "Ye, Yongshi and
Fu, Biao and
Chen, Xinjie and
Liao, Minpeng and
Shi, Xiaodong",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2553/",
pages = "58038--58059",
ISBN = "979-8-89176-459-0",
abstract = "Reinforcement learning with verifiable rewards (RLVR) improves the reasoning ability of large language models (LLMs), but its standard form relies on ground-truth answers or external verifiers that are often unavailable in unlabeled test-time settings. Label-free RLVR therefore estimates pseudo rewards from model outputs. Existing methods typically use these pseudo rewards to optimize either answers or model parameters. The former cannot preserve feedback across queries, while the latter risks committing noisy signals to model parameters. This raises a central question: what should label-free rewards optimize? We propose Test-Time Memory Learning (TTML), a label-free RLVR framework that shifts the optimization target from answers or parameters to an external memory bank. TTML consists of three components: memory policy rollout selects useful memories and executes memory-augmented generation for the current query, Bayesian value update turns consensus rewards into memory value updates, and memory bank expansion extracts reusable experience from test-time trajectories. Across six benchmarks and three backbone models, TTML achieves the best Avg@8 performance over strong label-free baselines. Empirical analyses further suggest that converting label-free feedback into revisable memory values and reusable memory entries is more effective overall than using it only for single-query selection or direct parameter updates."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ye-etal-2026-ttml">
<titleInfo>
<title>TTML: Test-Time Memory Learning for Label-Free RLVR</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yongshi</namePart>
<namePart type="family">Ye</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Biao</namePart>
<namePart type="family">Fu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minpeng</namePart>
<namePart type="family">Liao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaodong</namePart>
<namePart type="family">Shi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Reinforcement learning with verifiable rewards (RLVR) improves the reasoning ability of large language models (LLMs), but its standard form relies on ground-truth answers or external verifiers that are often unavailable in unlabeled test-time settings. Label-free RLVR therefore estimates pseudo rewards from model outputs. Existing methods typically use these pseudo rewards to optimize either answers or model parameters. The former cannot preserve feedback across queries, while the latter risks committing noisy signals to model parameters. This raises a central question: what should label-free rewards optimize? We propose Test-Time Memory Learning (TTML), a label-free RLVR framework that shifts the optimization target from answers or parameters to an external memory bank. TTML consists of three components: memory policy rollout selects useful memories and executes memory-augmented generation for the current query, Bayesian value update turns consensus rewards into memory value updates, and memory bank expansion extracts reusable experience from test-time trajectories. Across six benchmarks and three backbone models, TTML achieves the best Avg@8 performance over strong label-free baselines. Empirical analyses further suggest that converting label-free feedback into revisable memory values and reusable memory entries is more effective overall than using it only for single-query selection or direct parameter updates.</abstract>
<identifier type="citekey">ye-etal-2026-ttml</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2553/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>58038</start>
<end>58059</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T TTML: Test-Time Memory Learning for Label-Free RLVR
%A Ye, Yongshi
%A Fu, Biao
%A Chen, Xinjie
%A Liao, Minpeng
%A Shi, Xiaodong
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F ye-etal-2026-ttml
%X Reinforcement learning with verifiable rewards (RLVR) improves the reasoning ability of large language models (LLMs), but its standard form relies on ground-truth answers or external verifiers that are often unavailable in unlabeled test-time settings. Label-free RLVR therefore estimates pseudo rewards from model outputs. Existing methods typically use these pseudo rewards to optimize either answers or model parameters. The former cannot preserve feedback across queries, while the latter risks committing noisy signals to model parameters. This raises a central question: what should label-free rewards optimize? We propose Test-Time Memory Learning (TTML), a label-free RLVR framework that shifts the optimization target from answers or parameters to an external memory bank. TTML consists of three components: memory policy rollout selects useful memories and executes memory-augmented generation for the current query, Bayesian value update turns consensus rewards into memory value updates, and memory bank expansion extracts reusable experience from test-time trajectories. Across six benchmarks and three backbone models, TTML achieves the best Avg@8 performance over strong label-free baselines. Empirical analyses further suggest that converting label-free feedback into revisable memory values and reusable memory entries is more effective overall than using it only for single-query selection or direct parameter updates.
%U https://aclanthology.org/2026.emnlp-main.2553/
%P 58038-58059
Markdown (Informal)
[TTML: Test-Time Memory Learning for Label-Free RLVR](https://aclanthology.org/2026.emnlp-main.2553/) (Ye et al., EMNLP 2026)
ACL
- Yongshi Ye, Biao Fu, Xinjie Chen, Minpeng Liao, and Xiaodong Shi. 2026. TTML: Test-Time Memory Learning for Label-Free RLVR. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 58038–58059, Budapest, Hungary. Association for Computational Linguistics.