@inproceedings{knight-etal-2026-reviews,
title = "What Can Reviews Reliably Tell Us About Books? A Per-Attribute Audit and Tiered Annotated Dataset Across 1.7{M} Titles",
author = "Knight, Samsun and
Bart, Yakov and
Tang, Ethan and
Gutman, Jory",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.562/",
pages = "12903--12934",
ISBN = "979-8-89176-459-0",
abstract = "Aggregated reader reviews are widely used as scalable proxies for book attributes in computational literary studies, recommendation, and narratological-benchmark design, but the field lacks a per-attribute audit of what review aggregates can reliably recover. We audit eight review-derived feature methods on 391 attributes, using the HumanReader survey corpus (Knight et al., 2026) as ground truth. Most apparent predictive performance is popularity-confounded. Only 34.9{\%} of evaluable attributes have a method that adds signal beyond a popularity proxy (popularity-conditioned incremental {\ensuremath{\Delta}}R{\texttwosuperior} {\ensuremath{>}} 0.01), and just 12 additionally pass held-out validation, multiplicity correction, and a stricter {\ensuremath{\Delta}}R{\texttwosuperior} {\ensuremath{>}} 0.05 release gate. Recoverability is significantly associated with an attribute{'}s ``lexical anchorability,'' i.e. how often reviews mention keywords in the attribute{'}s semantic neighborhood, with high-anchorability attributes about 2.3 times more likely to clear validation (50{\%} of the top anchorability quartile, 22{\%} of the bottom). We release HumanReader-Reviews-1.7M, per-book predictions on 12 content-driven attributes (Tier A) and 77 validated but popularity-informed attributes (Tier B), with per-method test statistics on the 111 attributes no scalable method recovers (Tier C). Raw held-out R{\texttwosuperior} is a liberal estimate of per-book content recovery, crediting any popularity-correlated content, while popularity-conditioned {\ensuremath{\Delta}}R{\texttwosuperior} is conservative in what it attributes to content; we report both. On a coverage-limited probe of 31 attributes, direct LLM-on-reviews annotation recovers signal the release-panel methods miss, but at a compute cost prohibiting per-book deployment across the 1.7M-book corpus; as a complement, we release an open-weight Qwen 2.5-32B companion scoring 9,990 books."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="knight-etal-2026-reviews">
<titleInfo>
<title>What Can Reviews Reliably Tell Us About Books? A Per-Attribute Audit and Tiered Annotated Dataset Across 1.7M Titles</title>
</titleInfo>
<name type="personal">
<namePart type="given">Samsun</namePart>
<namePart type="family">Knight</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yakov</namePart>
<namePart type="family">Bart</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jory</namePart>
<namePart type="family">Gutman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Aggregated reader reviews are widely used as scalable proxies for book attributes in computational literary studies, recommendation, and narratological-benchmark design, but the field lacks a per-attribute audit of what review aggregates can reliably recover. We audit eight review-derived feature methods on 391 attributes, using the HumanReader survey corpus (Knight et al., 2026) as ground truth. Most apparent predictive performance is popularity-confounded. Only 34.9% of evaluable attributes have a method that adds signal beyond a popularity proxy (popularity-conditioned incremental \ensuremathΔR² \ensuremath> 0.01), and just 12 additionally pass held-out validation, multiplicity correction, and a stricter \ensuremathΔR² \ensuremath> 0.05 release gate. Recoverability is significantly associated with an attribute’s “lexical anchorability,” i.e. how often reviews mention keywords in the attribute’s semantic neighborhood, with high-anchorability attributes about 2.3 times more likely to clear validation (50% of the top anchorability quartile, 22% of the bottom). We release HumanReader-Reviews-1.7M, per-book predictions on 12 content-driven attributes (Tier A) and 77 validated but popularity-informed attributes (Tier B), with per-method test statistics on the 111 attributes no scalable method recovers (Tier C). Raw held-out R² is a liberal estimate of per-book content recovery, crediting any popularity-correlated content, while popularity-conditioned \ensuremathΔR² is conservative in what it attributes to content; we report both. On a coverage-limited probe of 31 attributes, direct LLM-on-reviews annotation recovers signal the release-panel methods miss, but at a compute cost prohibiting per-book deployment across the 1.7M-book corpus; as a complement, we release an open-weight Qwen 2.5-32B companion scoring 9,990 books.</abstract>
<identifier type="citekey">knight-etal-2026-reviews</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.562/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>12903</start>
<end>12934</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Can Reviews Reliably Tell Us About Books? A Per-Attribute Audit and Tiered Annotated Dataset Across 1.7M Titles
%A Knight, Samsun
%A Bart, Yakov
%A Tang, Ethan
%A Gutman, Jory
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F knight-etal-2026-reviews
%X Aggregated reader reviews are widely used as scalable proxies for book attributes in computational literary studies, recommendation, and narratological-benchmark design, but the field lacks a per-attribute audit of what review aggregates can reliably recover. We audit eight review-derived feature methods on 391 attributes, using the HumanReader survey corpus (Knight et al., 2026) as ground truth. Most apparent predictive performance is popularity-confounded. Only 34.9% of evaluable attributes have a method that adds signal beyond a popularity proxy (popularity-conditioned incremental \ensuremathΔR² \ensuremath> 0.01), and just 12 additionally pass held-out validation, multiplicity correction, and a stricter \ensuremathΔR² \ensuremath> 0.05 release gate. Recoverability is significantly associated with an attribute’s “lexical anchorability,” i.e. how often reviews mention keywords in the attribute’s semantic neighborhood, with high-anchorability attributes about 2.3 times more likely to clear validation (50% of the top anchorability quartile, 22% of the bottom). We release HumanReader-Reviews-1.7M, per-book predictions on 12 content-driven attributes (Tier A) and 77 validated but popularity-informed attributes (Tier B), with per-method test statistics on the 111 attributes no scalable method recovers (Tier C). Raw held-out R² is a liberal estimate of per-book content recovery, crediting any popularity-correlated content, while popularity-conditioned \ensuremathΔR² is conservative in what it attributes to content; we report both. On a coverage-limited probe of 31 attributes, direct LLM-on-reviews annotation recovers signal the release-panel methods miss, but at a compute cost prohibiting per-book deployment across the 1.7M-book corpus; as a complement, we release an open-weight Qwen 2.5-32B companion scoring 9,990 books.
%U https://aclanthology.org/2026.emnlp-main.562/
%P 12903-12934
Markdown (Informal)
[What Can Reviews Reliably Tell Us About Books? A Per-Attribute Audit and Tiered Annotated Dataset Across 1.7M Titles](https://aclanthology.org/2026.emnlp-main.562/) (Knight et al., EMNLP 2026)
ACL