@inproceedings{wang-etal-2026-decoupled,
title = "Decoupled Analysis-Judging: An Automated Creativity Evaluator Using {LLM}s in Complex Multi-step Creativity Tasks",
author = "Wang, Xiangyu and
Wu, Jin and
Li, Xiaoyu and
Zheng, Chanjin and
Zhou, Yifeng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.816/",
pages = "17272--17308",
ISBN = "979-8-89176-462-0",
abstract = "Automated evaluation of creativity tasks remains challenging for LLM-as-a-Judge, as LLM is susceptible to biases such as verbosity bias and leniency bias. Such limitations are particularly evident in Contextually-Grounded and Procedurally-Structured Tasks (CGPST), a complex multi-step creativity task where inter-step dependencies, highly subjectivity, and wide scoring ranges lead to more unstable and biased judgments. Existing approaches either rely on task-specific training or directly apply LLM-as-a-Judge, both of which struggle to ensure reliable evaluation under such complexity. To bridge these gaps, we propose **CreaEval**, an automated **crea**tivity **eval**uator for CGPST that decouples typical LLM-as-a-Judge into analysis and judging. Correspondingly, CreaEval involves two critical phases: **Memory-augmented Analysis**, a SoT-LLM converts multi-step responses into structured evaluation evidence, incorporating cross-step memory; and **Evidence-based Judging**, a Judge-LLM uses the extracted evidence for judging without accessing raw responses. Comprehensive experiments on CGPST show that CreaEval achieves an average human-LLM agreement of 0.64 (QWK) on CGPST, improving over the second-best baseline (supervised training) by 0.17. The code is available at https://anonymous.4open.science/r/CreaEval-0058/."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-etal-2026-decoupled">
<titleInfo>
<title>Decoupled Analysis-Judging: An Automated Creativity Evaluator Using LLMs in Complex Multi-step Creativity Tasks</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xiangyu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jin</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaoyu</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chanjin</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yifeng</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Automated evaluation of creativity tasks remains challenging for LLM-as-a-Judge, as LLM is susceptible to biases such as verbosity bias and leniency bias. Such limitations are particularly evident in Contextually-Grounded and Procedurally-Structured Tasks (CGPST), a complex multi-step creativity task where inter-step dependencies, highly subjectivity, and wide scoring ranges lead to more unstable and biased judgments. Existing approaches either rely on task-specific training or directly apply LLM-as-a-Judge, both of which struggle to ensure reliable evaluation under such complexity. To bridge these gaps, we propose **CreaEval**, an automated **crea**tivity **eval**uator for CGPST that decouples typical LLM-as-a-Judge into analysis and judging. Correspondingly, CreaEval involves two critical phases: **Memory-augmented Analysis**, a SoT-LLM converts multi-step responses into structured evaluation evidence, incorporating cross-step memory; and **Evidence-based Judging**, a Judge-LLM uses the extracted evidence for judging without accessing raw responses. Comprehensive experiments on CGPST show that CreaEval achieves an average human-LLM agreement of 0.64 (QWK) on CGPST, improving over the second-best baseline (supervised training) by 0.17. The code is available at https://anonymous.4open.science/r/CreaEval-0058/.</abstract>
<identifier type="citekey">wang-etal-2026-decoupled</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.816/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>17272</start>
<end>17308</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Decoupled Analysis-Judging: An Automated Creativity Evaluator Using LLMs in Complex Multi-step Creativity Tasks
%A Wang, Xiangyu
%A Wu, Jin
%A Li, Xiaoyu
%A Zheng, Chanjin
%A Zhou, Yifeng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F wang-etal-2026-decoupled
%X Automated evaluation of creativity tasks remains challenging for LLM-as-a-Judge, as LLM is susceptible to biases such as verbosity bias and leniency bias. Such limitations are particularly evident in Contextually-Grounded and Procedurally-Structured Tasks (CGPST), a complex multi-step creativity task where inter-step dependencies, highly subjectivity, and wide scoring ranges lead to more unstable and biased judgments. Existing approaches either rely on task-specific training or directly apply LLM-as-a-Judge, both of which struggle to ensure reliable evaluation under such complexity. To bridge these gaps, we propose **CreaEval**, an automated **crea**tivity **eval**uator for CGPST that decouples typical LLM-as-a-Judge into analysis and judging. Correspondingly, CreaEval involves two critical phases: **Memory-augmented Analysis**, a SoT-LLM converts multi-step responses into structured evaluation evidence, incorporating cross-step memory; and **Evidence-based Judging**, a Judge-LLM uses the extracted evidence for judging without accessing raw responses. Comprehensive experiments on CGPST show that CreaEval achieves an average human-LLM agreement of 0.64 (QWK) on CGPST, improving over the second-best baseline (supervised training) by 0.17. The code is available at https://anonymous.4open.science/r/CreaEval-0058/.
%U https://aclanthology.org/2026.findings-emnlp.816/
%P 17272-17308
Markdown (Informal)
[Decoupled Analysis-Judging: An Automated Creativity Evaluator Using LLMs in Complex Multi-step Creativity Tasks](https://aclanthology.org/2026.findings-emnlp.816/) (Wang et al., Findings 2026)
ACL