@inproceedings{wu-etal-2026-collabmem,
title = "{COLLABMEM}: A Diagnostic Benchmark for Collaborative Memory Operations in Multi-Agent {LLM} Systems",
author = "Wu, Shun and
Wu, Di and
Zhang, XueYou and
Liu, Kang",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1194/",
pages = "27343--27356",
ISBN = "979-8-89176-459-0",
abstract = "Multi-agent LLM systems increasingly rely on persistent memory to coordinate role-specialized work. Correctness requires more than recalling a fact: the fact must reach the right role, remain hidden from unauthorized roles, carry the right source, and update when a more authoritative value appears. We introduce COLLABMEM, a diagnostic benchmark for collaborative memory operations. Its matched memory-only harness fixes the trace, target role, answerer, judge, and gold event, localizing errors to sharing, access, routing, provenance, conflict resolution, or retention. COLLABMEM contains 58 hand-authored scenarios (1,118 core probes), a 174-scenario scaled split (3,359 probes), adversarial and compositional hard sets, and six transactive-memory competencies. We also propose TMS-GRAPH++, a scope-aware graph memory framework with role-filtered views, writer/provenance structure, and temporal [CURRENT] marking. Across five evaluation splits, TMS-GRAPH++ achieves the highest non-Oracle average score (69.6{\%}). Evaluations using the official SDKs of Mem0, Letta, and A-Mem show that shared archives help, but reliable collaborative memory still requires explicit visibility, provenance, authority, and retention policies. We will release the dataset, evaluation code, and reproduced outputs to support future work on multi-agent memory systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wu-etal-2026-collabmem">
<titleInfo>
<title>COLLABMEM: A Diagnostic Benchmark for Collaborative Memory Operations in Multi-Agent LLM Systems</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shun</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">XueYou</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kang</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Multi-agent LLM systems increasingly rely on persistent memory to coordinate role-specialized work. Correctness requires more than recalling a fact: the fact must reach the right role, remain hidden from unauthorized roles, carry the right source, and update when a more authoritative value appears. We introduce COLLABMEM, a diagnostic benchmark for collaborative memory operations. Its matched memory-only harness fixes the trace, target role, answerer, judge, and gold event, localizing errors to sharing, access, routing, provenance, conflict resolution, or retention. COLLABMEM contains 58 hand-authored scenarios (1,118 core probes), a 174-scenario scaled split (3,359 probes), adversarial and compositional hard sets, and six transactive-memory competencies. We also propose TMS-GRAPH++, a scope-aware graph memory framework with role-filtered views, writer/provenance structure, and temporal [CURRENT] marking. Across five evaluation splits, TMS-GRAPH++ achieves the highest non-Oracle average score (69.6%). Evaluations using the official SDKs of Mem0, Letta, and A-Mem show that shared archives help, but reliable collaborative memory still requires explicit visibility, provenance, authority, and retention policies. We will release the dataset, evaluation code, and reproduced outputs to support future work on multi-agent memory systems.</abstract>
<identifier type="citekey">wu-etal-2026-collabmem</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1194/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>27343</start>
<end>27356</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T COLLABMEM: A Diagnostic Benchmark for Collaborative Memory Operations in Multi-Agent LLM Systems
%A Wu, Shun
%A Wu, Di
%A Zhang, XueYou
%A Liu, Kang
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F wu-etal-2026-collabmem
%X Multi-agent LLM systems increasingly rely on persistent memory to coordinate role-specialized work. Correctness requires more than recalling a fact: the fact must reach the right role, remain hidden from unauthorized roles, carry the right source, and update when a more authoritative value appears. We introduce COLLABMEM, a diagnostic benchmark for collaborative memory operations. Its matched memory-only harness fixes the trace, target role, answerer, judge, and gold event, localizing errors to sharing, access, routing, provenance, conflict resolution, or retention. COLLABMEM contains 58 hand-authored scenarios (1,118 core probes), a 174-scenario scaled split (3,359 probes), adversarial and compositional hard sets, and six transactive-memory competencies. We also propose TMS-GRAPH++, a scope-aware graph memory framework with role-filtered views, writer/provenance structure, and temporal [CURRENT] marking. Across five evaluation splits, TMS-GRAPH++ achieves the highest non-Oracle average score (69.6%). Evaluations using the official SDKs of Mem0, Letta, and A-Mem show that shared archives help, but reliable collaborative memory still requires explicit visibility, provenance, authority, and retention policies. We will release the dataset, evaluation code, and reproduced outputs to support future work on multi-agent memory systems.
%U https://aclanthology.org/2026.emnlp-main.1194/
%P 27343-27356
Markdown (Informal)
[COLLABMEM: A Diagnostic Benchmark for Collaborative Memory Operations in Multi-Agent LLM Systems](https://aclanthology.org/2026.emnlp-main.1194/) (Wu et al., EMNLP 2026)
ACL