@inproceedings{yi-etal-2026-poetry,
title = "From Poetry Parties to Poetry Critics: Benchmarking Classical {C}hinese Poetry Generation and Evaluation",
author = "Yi, Zihao and
Tu, Zhaopeng and
Ma, Ruotian and
Jiang, Qingxuan and
Xu, Zhe and
Xiao, Qingyu and
Li, Yixue and
Shen, Ying and
Bo, Liefeng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1534/",
pages = "35021--35036",
ISBN = "979-8-89176-459-0",
abstract = "Classical Chinese poetry generation and evaluation with Large Language Models (LLMs) face two intertwined challenges: existing generation benchmarks rely on simplistic keyword inputs devoid of literary context, and no ground-truth benchmark exists for objectively validating poetry evaluation methods. We present a comprehensive framework addressing both challenges. On the **generation** side, we construct **Honglou-Poem**, the first narrative-driven poetry generation benchmark derived from Dream of the Red Chamber, where each entry provides rich author personas and narrative backgrounds{---}moving far beyond traditional keyword-to-poem paradigms. A ``Digital Poetry Party'' pipeline evaluates 12 state-of-the-art LLMs in this challenging setting. On the **evaluation** side, we introduce **Poetry-Tree**, a benchmark with axiomatically ordered quality levels created through systematic merit degradation, providing objective ground truth for evaluator validation. Leveraging Poetry-Tree as a training signal, we employ **GRPO** reinforcement learning with Spearman{'}s correlation as the reward to train a dedicated poetry evaluation model, **Poet-4B**. Our Poet-4B achieves a Spearman{'}s $\rho$ of **0.8253**, dramatically improving from the base model{'}s 0.2929 and closely approaching the best closed-source model GLM-4.6 (0.8764) with over $100{\times}$ fewer parameters{---}a gap of only 0.051 in Spearman{'}s $\rho$. We release all benchmarks, trained models, and a leaderboard to facilitate future research."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="yi-etal-2026-poetry">
<titleInfo>
<title>From Poetry Parties to Poetry Critics: Benchmarking Classical Chinese Poetry Generation and Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zihao</namePart>
<namePart type="family">Yi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhaopeng</namePart>
<namePart type="family">Tu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruotian</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qingxuan</namePart>
<namePart type="family">Jiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhe</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qingyu</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yixue</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ying</namePart>
<namePart type="family">Shen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Liefeng</namePart>
<namePart type="family">Bo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Classical Chinese poetry generation and evaluation with Large Language Models (LLMs) face two intertwined challenges: existing generation benchmarks rely on simplistic keyword inputs devoid of literary context, and no ground-truth benchmark exists for objectively validating poetry evaluation methods. We present a comprehensive framework addressing both challenges. On the **generation** side, we construct **Honglou-Poem**, the first narrative-driven poetry generation benchmark derived from Dream of the Red Chamber, where each entry provides rich author personas and narrative backgrounds—moving far beyond traditional keyword-to-poem paradigms. A “Digital Poetry Party” pipeline evaluates 12 state-of-the-art LLMs in this challenging setting. On the **evaluation** side, we introduce **Poetry-Tree**, a benchmark with axiomatically ordered quality levels created through systematic merit degradation, providing objective ground truth for evaluator validation. Leveraging Poetry-Tree as a training signal, we employ **GRPO** reinforcement learning with Spearman’s correlation as the reward to train a dedicated poetry evaluation model, **Poet-4B**. Our Poet-4B achieves a Spearman’s ρ of **0.8253**, dramatically improving from the base model’s 0.2929 and closely approaching the best closed-source model GLM-4.6 (0.8764) with over 100\times fewer parameters—a gap of only 0.051 in Spearman’s ρ. We release all benchmarks, trained models, and a leaderboard to facilitate future research.</abstract>
<identifier type="citekey">yi-etal-2026-poetry</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1534/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>35021</start>
<end>35036</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T From Poetry Parties to Poetry Critics: Benchmarking Classical Chinese Poetry Generation and Evaluation
%A Yi, Zihao
%A Tu, Zhaopeng
%A Ma, Ruotian
%A Jiang, Qingxuan
%A Xu, Zhe
%A Xiao, Qingyu
%A Li, Yixue
%A Shen, Ying
%A Bo, Liefeng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F yi-etal-2026-poetry
%X Classical Chinese poetry generation and evaluation with Large Language Models (LLMs) face two intertwined challenges: existing generation benchmarks rely on simplistic keyword inputs devoid of literary context, and no ground-truth benchmark exists for objectively validating poetry evaluation methods. We present a comprehensive framework addressing both challenges. On the **generation** side, we construct **Honglou-Poem**, the first narrative-driven poetry generation benchmark derived from Dream of the Red Chamber, where each entry provides rich author personas and narrative backgrounds—moving far beyond traditional keyword-to-poem paradigms. A “Digital Poetry Party” pipeline evaluates 12 state-of-the-art LLMs in this challenging setting. On the **evaluation** side, we introduce **Poetry-Tree**, a benchmark with axiomatically ordered quality levels created through systematic merit degradation, providing objective ground truth for evaluator validation. Leveraging Poetry-Tree as a training signal, we employ **GRPO** reinforcement learning with Spearman’s correlation as the reward to train a dedicated poetry evaluation model, **Poet-4B**. Our Poet-4B achieves a Spearman’s ρ of **0.8253**, dramatically improving from the base model’s 0.2929 and closely approaching the best closed-source model GLM-4.6 (0.8764) with over 100\times fewer parameters—a gap of only 0.051 in Spearman’s ρ. We release all benchmarks, trained models, and a leaderboard to facilitate future research.
%U https://aclanthology.org/2026.emnlp-main.1534/
%P 35021-35036
Markdown (Informal)
[From Poetry Parties to Poetry Critics: Benchmarking Classical Chinese Poetry Generation and Evaluation](https://aclanthology.org/2026.emnlp-main.1534/) (Yi et al., EMNLP 2026)
ACL
- Zihao Yi, Zhaopeng Tu, Ruotian Ma, Qingxuan Jiang, Zhe Xu, Qingyu Xiao, Yixue Li, Ying Shen, and Liefeng Bo. 2026. From Poetry Parties to Poetry Critics: Benchmarking Classical Chinese Poetry Generation and Evaluation. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 35021–35036, Budapest, Hungary. Association for Computational Linguistics.