@inproceedings{kuznetsov-etal-2026-peer,
title = "Is Peer Review Really in Decline? Analyzing Review Quality across Venues and Time",
author = "Kuznetsov, Ilia and
Nayak, Rohan Kumar and
Rozovskaya, Alla and
Gurevych, Iryna",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1273/",
pages = "29017--29056",
ISBN = "979-8-89176-459-0",
abstract = "Peer review is at the heart of modern science. As submission numbers rise and research communities grow, the decline in review quality is a popular narrative and a common concern. Yet, is it true? Review quality is difficult to measure, and the ongoing evolution of reviewing practices makes it difficult to compare reviews across venues and time. To address this, we introduce a new framework for the comparative study of review quality and apply it to major AI and machine learning conferences. We propose a multi-dimensional schema to serve as a proxy for quantifying review quality in terms of its utility to editors and authors. The schema is coupled with LLM-based and lightweight measurements, which we validate against a novel test set of human-labeled review quality ratings. Contradicting the popular narrative, our proxy measurements reveal no consistent decline in median review quality across venues and years. We propose alternative explanations, and outline recommendations to facilitate future empirical studies of review quality. Code and data are available at https://github.com/UKPLab/emnlp2026-review-quality-estimation."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kuznetsov-etal-2026-peer">
<titleInfo>
<title>Is Peer Review Really in Decline? Analyzing Review Quality across Venues and Time</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ilia</namePart>
<namePart type="family">Kuznetsov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rohan</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Nayak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alla</namePart>
<namePart type="family">Rozovskaya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Iryna</namePart>
<namePart type="family">Gurevych</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Peer review is at the heart of modern science. As submission numbers rise and research communities grow, the decline in review quality is a popular narrative and a common concern. Yet, is it true? Review quality is difficult to measure, and the ongoing evolution of reviewing practices makes it difficult to compare reviews across venues and time. To address this, we introduce a new framework for the comparative study of review quality and apply it to major AI and machine learning conferences. We propose a multi-dimensional schema to serve as a proxy for quantifying review quality in terms of its utility to editors and authors. The schema is coupled with LLM-based and lightweight measurements, which we validate against a novel test set of human-labeled review quality ratings. Contradicting the popular narrative, our proxy measurements reveal no consistent decline in median review quality across venues and years. We propose alternative explanations, and outline recommendations to facilitate future empirical studies of review quality. Code and data are available at https://github.com/UKPLab/emnlp2026-review-quality-estimation.</abstract>
<identifier type="citekey">kuznetsov-etal-2026-peer</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1273/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>29017</start>
<end>29056</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Is Peer Review Really in Decline? Analyzing Review Quality across Venues and Time
%A Kuznetsov, Ilia
%A Nayak, Rohan Kumar
%A Rozovskaya, Alla
%A Gurevych, Iryna
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kuznetsov-etal-2026-peer
%X Peer review is at the heart of modern science. As submission numbers rise and research communities grow, the decline in review quality is a popular narrative and a common concern. Yet, is it true? Review quality is difficult to measure, and the ongoing evolution of reviewing practices makes it difficult to compare reviews across venues and time. To address this, we introduce a new framework for the comparative study of review quality and apply it to major AI and machine learning conferences. We propose a multi-dimensional schema to serve as a proxy for quantifying review quality in terms of its utility to editors and authors. The schema is coupled with LLM-based and lightweight measurements, which we validate against a novel test set of human-labeled review quality ratings. Contradicting the popular narrative, our proxy measurements reveal no consistent decline in median review quality across venues and years. We propose alternative explanations, and outline recommendations to facilitate future empirical studies of review quality. Code and data are available at https://github.com/UKPLab/emnlp2026-review-quality-estimation.
%U https://aclanthology.org/2026.emnlp-main.1273/
%P 29017-29056
Markdown (Informal)
[Is Peer Review Really in Decline? Analyzing Review Quality across Venues and Time](https://aclanthology.org/2026.emnlp-main.1273/) (Kuznetsov et al., EMNLP 2026)
ACL