@inproceedings{sathya-etal-2026-framework,
title = "A Framework for Evaluating {AI}-Generated Creativity: Human{--}{LLM} Agreement in Evaluating {AI}-Generated Stories",
author = "Sathya, Pharath and
Huang, Yin Jou and
Cheng, Fei",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.203/",
pages = "4480--4502",
ISBN = "979-8-89176-439-2",
abstract = "Creative text is inherently subjective: any opinion can be equally valid, and there is no objective ground truth. The only way to quantify such subjectivity is to model the crowd of opinions that produces it. Large language model (LLM)-as-a-judge has emerged as a scalable substitute for human raters, but whether an LLM can stand in for the human crowd on AI creativity remains unexplored. We develop a decomposition framework (four main components, 11 sub-components) and a three-step rating procedure, applied identically to human raters and to different LLM judging conditions on AI-generated stories. Humans rate AI creativity with the same subjectivity that creativity research documents for human-authored creative work. Across three structural comparisons (disagreement on the same story including ranking; revision from impressionistic to reflective; sub-component weighting of the final rating), no LLM reproduces all three human patterns. We frame the contribution as a methodological framework plus a quantification of where LLM judges currently fall short of the human ratings in our study. We release all stories, ratings, and code at https://github.com/ku-nlp/creativity-evaluation-artifact."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sathya-etal-2026-framework">
<titleInfo>
<title>A Framework for Evaluating AI-Generated Creativity: Human–LLM Agreement in Evaluating AI-Generated Stories</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pharath</namePart>
<namePart type="family">Sathya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yin</namePart>
<namePart type="given">Jou</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fei</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Creative text is inherently subjective: any opinion can be equally valid, and there is no objective ground truth. The only way to quantify such subjectivity is to model the crowd of opinions that produces it. Large language model (LLM)-as-a-judge has emerged as a scalable substitute for human raters, but whether an LLM can stand in for the human crowd on AI creativity remains unexplored. We develop a decomposition framework (four main components, 11 sub-components) and a three-step rating procedure, applied identically to human raters and to different LLM judging conditions on AI-generated stories. Humans rate AI creativity with the same subjectivity that creativity research documents for human-authored creative work. Across three structural comparisons (disagreement on the same story including ranking; revision from impressionistic to reflective; sub-component weighting of the final rating), no LLM reproduces all three human patterns. We frame the contribution as a methodological framework plus a quantification of where LLM judges currently fall short of the human ratings in our study. We release all stories, ratings, and code at https://github.com/ku-nlp/creativity-evaluation-artifact.</abstract>
<identifier type="citekey">sathya-etal-2026-framework</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.203/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4480</start>
<end>4502</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Framework for Evaluating AI-Generated Creativity: Human–LLM Agreement in Evaluating AI-Generated Stories
%A Sathya, Pharath
%A Huang, Yin Jou
%A Cheng, Fei
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F sathya-etal-2026-framework
%X Creative text is inherently subjective: any opinion can be equally valid, and there is no objective ground truth. The only way to quantify such subjectivity is to model the crowd of opinions that produces it. Large language model (LLM)-as-a-judge has emerged as a scalable substitute for human raters, but whether an LLM can stand in for the human crowd on AI creativity remains unexplored. We develop a decomposition framework (four main components, 11 sub-components) and a three-step rating procedure, applied identically to human raters and to different LLM judging conditions on AI-generated stories. Humans rate AI creativity with the same subjectivity that creativity research documents for human-authored creative work. Across three structural comparisons (disagreement on the same story including ranking; revision from impressionistic to reflective; sub-component weighting of the final rating), no LLM reproduces all three human patterns. We frame the contribution as a methodological framework plus a quantification of where LLM judges currently fall short of the human ratings in our study. We release all stories, ratings, and code at https://github.com/ku-nlp/creativity-evaluation-artifact.
%U https://aclanthology.org/2026.aacl-long.203/
%P 4480-4502
Markdown (Informal)
[A Framework for Evaluating AI-Generated Creativity: Human–LLM Agreement in Evaluating AI-Generated Stories](https://aclanthology.org/2026.aacl-long.203/) (Sathya et al., AACL-IJCNLP 2026)
ACL