@inproceedings{chen-etal-2026-prereviewbench,
title = "{PREREVIEWBENCH}: Benchmarking {LLM}s for Editorial Desk Screening in Scholarly Publishing",
author = "Chen, Ming and
Ning, Zhiyuan and
Du, Yi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1869/",
pages = "42593--42614",
ISBN = "979-8-89176-459-0",
abstract = "Before any reviewer is assigned, a journal editor decides whether a manuscript merits external review. Yet this routine desk screening has no benchmark. We present PREREVIEWBENCH, the first controlled benchmark for journal desk screening. Because real desk decisions are confidential, we pair Web of Science papers published in 2025 with journals to construct proxy labels: true matches, within-field caliber surpluses and deficits, and cross-field scope mismatches. The 7,292 balanced paper{--}journal instances span four broad areas and 58 journals, supporting desk triage, branch-specific assessment (caliber fit or rejection cause), and transfer-journal recommendation. Evaluating four frontier LLM families, we find average triage accuracy of 0.965 on cross-field scope mismatches but only 0.572 on within-field caliber deficits, where blind expert-led review reproduces the triage labels perfectly. Transfer recommendation is weaker still, with the best Hit@1 at just 0.369. Models are also sensitive to input perturbations, and a cue suggesting that most papers merit review sharply shifts GPT-5.5{'}s decisions. These results point to a clear division of labor: LLMs can already carry the scope side of desk screening, while caliber judgment stays with editors. PREREVIEWBENCH makes that harder half measurable: whether an in-scope paper is strong enough for its journal."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="chen-etal-2026-prereviewbench">
<titleInfo>
<title>PREREVIEWBENCH: Benchmarking LLMs for Editorial Desk Screening in Scholarly Publishing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ming</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhiyuan</namePart>
<namePart type="family">Ning</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="family">Du</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Before any reviewer is assigned, a journal editor decides whether a manuscript merits external review. Yet this routine desk screening has no benchmark. We present PREREVIEWBENCH, the first controlled benchmark for journal desk screening. Because real desk decisions are confidential, we pair Web of Science papers published in 2025 with journals to construct proxy labels: true matches, within-field caliber surpluses and deficits, and cross-field scope mismatches. The 7,292 balanced paper–journal instances span four broad areas and 58 journals, supporting desk triage, branch-specific assessment (caliber fit or rejection cause), and transfer-journal recommendation. Evaluating four frontier LLM families, we find average triage accuracy of 0.965 on cross-field scope mismatches but only 0.572 on within-field caliber deficits, where blind expert-led review reproduces the triage labels perfectly. Transfer recommendation is weaker still, with the best Hit@1 at just 0.369. Models are also sensitive to input perturbations, and a cue suggesting that most papers merit review sharply shifts GPT-5.5’s decisions. These results point to a clear division of labor: LLMs can already carry the scope side of desk screening, while caliber judgment stays with editors. PREREVIEWBENCH makes that harder half measurable: whether an in-scope paper is strong enough for its journal.</abstract>
<identifier type="citekey">chen-etal-2026-prereviewbench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1869/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>42593</start>
<end>42614</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PREREVIEWBENCH: Benchmarking LLMs for Editorial Desk Screening in Scholarly Publishing
%A Chen, Ming
%A Ning, Zhiyuan
%A Du, Yi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F chen-etal-2026-prereviewbench
%X Before any reviewer is assigned, a journal editor decides whether a manuscript merits external review. Yet this routine desk screening has no benchmark. We present PREREVIEWBENCH, the first controlled benchmark for journal desk screening. Because real desk decisions are confidential, we pair Web of Science papers published in 2025 with journals to construct proxy labels: true matches, within-field caliber surpluses and deficits, and cross-field scope mismatches. The 7,292 balanced paper–journal instances span four broad areas and 58 journals, supporting desk triage, branch-specific assessment (caliber fit or rejection cause), and transfer-journal recommendation. Evaluating four frontier LLM families, we find average triage accuracy of 0.965 on cross-field scope mismatches but only 0.572 on within-field caliber deficits, where blind expert-led review reproduces the triage labels perfectly. Transfer recommendation is weaker still, with the best Hit@1 at just 0.369. Models are also sensitive to input perturbations, and a cue suggesting that most papers merit review sharply shifts GPT-5.5’s decisions. These results point to a clear division of labor: LLMs can already carry the scope side of desk screening, while caliber judgment stays with editors. PREREVIEWBENCH makes that harder half measurable: whether an in-scope paper is strong enough for its journal.
%U https://aclanthology.org/2026.emnlp-main.1869/
%P 42593-42614
Markdown (Informal)
[PREREVIEWBENCH: Benchmarking LLMs for Editorial Desk Screening in Scholarly Publishing](https://aclanthology.org/2026.emnlp-main.1869/) (Chen et al., EMNLP 2026)
ACL