@inproceedings{mao-etal-2026-historical,
title = "From Historical Queries to Actionable Rubrics: Category-Aware Rubric Generation for Local-Life {AI} Search",
author = "Mao, Jiale and
Zhang, Fuwei and
Xi, Dongbo and
Chen, Zhen and
Yang, Yifan and
Yan, Peng and
Zhang, Zhao and
Zhuang, Fuzhen",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.65/",
pages = "1091--1110",
ISBN = "979-8-89176-460-6",
abstract = "Rubric generation improves the interpretability and actionability of large language model evaluation, but in online-to-offline (O2O) local-life AI search, user queries often reflect more complex needs, such as consumption intent, location constraints, and scenario preferences. Directly generating rubrics from a single query tends to produce generic criteria that fail to satisfy AI search evaluation needs, while leveraging candidate responses, preference samples, or external evidence at scale remains challenging. To address these limitations, we propose Category-Aware Rubric Generation (CaRG), a category-aware evaluation dimension candidate pool framework. Through embedding-based clustering, representative selection, and iterative LLM-based consolidation, CaRG constructs reusable candidate pools of both general and category-specific evaluation dimensions from representative historical queries. Subsequently, these candidate pools are leveraged to generate query-level rubrics with concrete positive and negative scoring criteria for target queries. Experimental results demonstrate that CaRG consistently outperforms Direct-Rubric and shows more stable overall quality than existing query-only baselines, while achieving the strongest overall agreement with production golden rubric rankings. Our repository is available at https://github.com/PlussLe/CaRG."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mao-etal-2026-historical">
<titleInfo>
<title>From Historical Queries to Actionable Rubrics: Category-Aware Rubric Generation for Local-Life AI Search</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiale</namePart>
<namePart type="family">Mao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fuwei</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dongbo</namePart>
<namePart type="family">Xi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhen</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifan</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peng</namePart>
<namePart type="family">Yan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fuzhen</namePart>
<namePart type="family">Zhuang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>Rubric generation improves the interpretability and actionability of large language model evaluation, but in online-to-offline (O2O) local-life AI search, user queries often reflect more complex needs, such as consumption intent, location constraints, and scenario preferences. Directly generating rubrics from a single query tends to produce generic criteria that fail to satisfy AI search evaluation needs, while leveraging candidate responses, preference samples, or external evidence at scale remains challenging. To address these limitations, we propose Category-Aware Rubric Generation (CaRG), a category-aware evaluation dimension candidate pool framework. Through embedding-based clustering, representative selection, and iterative LLM-based consolidation, CaRG constructs reusable candidate pools of both general and category-specific evaluation dimensions from representative historical queries. Subsequently, these candidate pools are leveraged to generate query-level rubrics with concrete positive and negative scoring criteria for target queries. Experimental results demonstrate that CaRG consistently outperforms Direct-Rubric and shows more stable overall quality than existing query-only baselines, while achieving the strongest overall agreement with production golden rubric rankings. Our repository is available at https://github.com/PlussLe/CaRG.</abstract>
<identifier type="citekey">mao-etal-2026-historical</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.65/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1091</start>
<end>1110</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T From Historical Queries to Actionable Rubrics: Category-Aware Rubric Generation for Local-Life AI Search
%A Mao, Jiale
%A Zhang, Fuwei
%A Xi, Dongbo
%A Chen, Zhen
%A Yang, Yifan
%A Yan, Peng
%A Zhang, Zhao
%A Zhuang, Fuzhen
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F mao-etal-2026-historical
%X Rubric generation improves the interpretability and actionability of large language model evaluation, but in online-to-offline (O2O) local-life AI search, user queries often reflect more complex needs, such as consumption intent, location constraints, and scenario preferences. Directly generating rubrics from a single query tends to produce generic criteria that fail to satisfy AI search evaluation needs, while leveraging candidate responses, preference samples, or external evidence at scale remains challenging. To address these limitations, we propose Category-Aware Rubric Generation (CaRG), a category-aware evaluation dimension candidate pool framework. Through embedding-based clustering, representative selection, and iterative LLM-based consolidation, CaRG constructs reusable candidate pools of both general and category-specific evaluation dimensions from representative historical queries. Subsequently, these candidate pools are leveraged to generate query-level rubrics with concrete positive and negative scoring criteria for target queries. Experimental results demonstrate that CaRG consistently outperforms Direct-Rubric and shows more stable overall quality than existing query-only baselines, while achieving the strongest overall agreement with production golden rubric rankings. Our repository is available at https://github.com/PlussLe/CaRG.
%U https://aclanthology.org/2026.emnlp-industry.65/
%P 1091-1110
Markdown (Informal)
[From Historical Queries to Actionable Rubrics: Category-Aware Rubric Generation for Local-Life AI Search](https://aclanthology.org/2026.emnlp-industry.65/) (Mao et al., EMNLP 2026)
ACL
- Jiale Mao, Fuwei Zhang, Dongbo Xi, Zhen Chen, Yifan Yang, Peng Yan, Zhao Zhang, and Fuzhen Zhuang. 2026. From Historical Queries to Actionable Rubrics: Category-Aware Rubric Generation for Local-Life AI Search. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 1091–1110, Budapest, Hungary. Association for Computational Linguistics.