@inproceedings{tan-etal-2026-beyond-codebook,
title = "Beyond Codebook Uniformity: Triplet-Level Metrics and Triplet-Driven Semantic {ID}s for Generative Retrieval",
author = "Tan, Xiang and
Liao, Dongliang and
Du, Junwu and
Wu, Haijun and
He, Run and
Man, Shuquan and
Zeng, Ziqian and
Zhuang, Huiping",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1914/",
pages = "43608--43622",
ISBN = "979-8-89176-459-0",
abstract = "Generative retrieval replaces item IDs with *Semantic IDs* (SIDs) decoded autoregressively by a sequence model. Playing the role of tokenizer, the codebook largely determines the quality of generative retrieval. Existing codebook metrics, however, only characterize the global uniformity of code usage and do not reveal whether an individual SID correctly represents its item or separates it from others. We address this problem with two metrics and a matching training objective. We build a query-centered triplet dataset $q, a, p, n$ from large-scale user behavior, and based on which we define **Overlap**, a prefix-based distance between two SIDs, and **Pairwise Overlap Ranking (POR)**, the fraction of triplets where a codebook places the positive closer to the anchor than the hard negative. Both metrics correlate strongly with downstream Hit Rate in generative retrieval. We then introduce the **Differentiable Soft Overlap Loss (DSOL)** to make these two metrics differentiable during codebook learning and optimize them directly. On a public e-commerce benchmark (ESCI-us) and a large-scale industrial video search dataset, DSOL improves HR and POR over standard SID baselines."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tan-etal-2026-beyond-codebook">
<titleInfo>
<title>Beyond Codebook Uniformity: Triplet-Level Metrics and Triplet-Driven Semantic IDs for Generative Retrieval</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Tan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dongliang</namePart>
<namePart type="family">Liao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junwu</namePart>
<namePart type="family">Du</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haijun</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Run</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuquan</namePart>
<namePart type="family">Man</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziqian</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Huiping</namePart>
<namePart type="family">Zhuang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Generative retrieval replaces item IDs with *Semantic IDs* (SIDs) decoded autoregressively by a sequence model. Playing the role of tokenizer, the codebook largely determines the quality of generative retrieval. Existing codebook metrics, however, only characterize the global uniformity of code usage and do not reveal whether an individual SID correctly represents its item or separates it from others. We address this problem with two metrics and a matching training objective. We build a query-centered triplet dataset q, a, p, n from large-scale user behavior, and based on which we define **Overlap**, a prefix-based distance between two SIDs, and **Pairwise Overlap Ranking (POR)**, the fraction of triplets where a codebook places the positive closer to the anchor than the hard negative. Both metrics correlate strongly with downstream Hit Rate in generative retrieval. We then introduce the **Differentiable Soft Overlap Loss (DSOL)** to make these two metrics differentiable during codebook learning and optimize them directly. On a public e-commerce benchmark (ESCI-us) and a large-scale industrial video search dataset, DSOL improves HR and POR over standard SID baselines.</abstract>
<identifier type="citekey">tan-etal-2026-beyond-codebook</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1914/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>43608</start>
<end>43622</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Codebook Uniformity: Triplet-Level Metrics and Triplet-Driven Semantic IDs for Generative Retrieval
%A Tan, Xiang
%A Liao, Dongliang
%A Du, Junwu
%A Wu, Haijun
%A He, Run
%A Man, Shuquan
%A Zeng, Ziqian
%A Zhuang, Huiping
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F tan-etal-2026-beyond-codebook
%X Generative retrieval replaces item IDs with *Semantic IDs* (SIDs) decoded autoregressively by a sequence model. Playing the role of tokenizer, the codebook largely determines the quality of generative retrieval. Existing codebook metrics, however, only characterize the global uniformity of code usage and do not reveal whether an individual SID correctly represents its item or separates it from others. We address this problem with two metrics and a matching training objective. We build a query-centered triplet dataset q, a, p, n from large-scale user behavior, and based on which we define **Overlap**, a prefix-based distance between two SIDs, and **Pairwise Overlap Ranking (POR)**, the fraction of triplets where a codebook places the positive closer to the anchor than the hard negative. Both metrics correlate strongly with downstream Hit Rate in generative retrieval. We then introduce the **Differentiable Soft Overlap Loss (DSOL)** to make these two metrics differentiable during codebook learning and optimize them directly. On a public e-commerce benchmark (ESCI-us) and a large-scale industrial video search dataset, DSOL improves HR and POR over standard SID baselines.
%U https://aclanthology.org/2026.emnlp-main.1914/
%P 43608-43622
Markdown (Informal)
[Beyond Codebook Uniformity: Triplet-Level Metrics and Triplet-Driven Semantic IDs for Generative Retrieval](https://aclanthology.org/2026.emnlp-main.1914/) (Tan et al., EMNLP 2026)
ACL
- Xiang Tan, Dongliang Liao, Junwu Du, Haijun Wu, Run He, Shuquan Man, Ziqian Zeng, and Huiping Zhuang. 2026. Beyond Codebook Uniformity: Triplet-Level Metrics and Triplet-Driven Semantic IDs for Generative Retrieval. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 43608–43622, Budapest, Hungary. Association for Computational Linguistics.