@inproceedings{rospocher-2026-contextual,
title = "Contextual Cue Sensitivity in {LLM} Evaluation of Grant Proposals: A Controlled {ERC}-Style Study",
author = "Rospocher, Marco",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 2: Short Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-short.44/",
pages = "679--698",
ISBN = "979-8-89176-440-8",
abstract = "Large language models (LLMs) are increasingly used as evaluators, but in high-stakes settings reliability requires more than consistency: scores should also be robust to contextual information that is not intended to alter the criterion being evaluated. We study this question in a controlled English-language ERC-style proposal-scoring setup. Starting from 40 Horizon Europe ERC-related project summaries, we construct synthetic Part B1-like proposal texts and evaluate them with eight LLMs under manipulations of principal-investigator (PI) profile information{---}name, institution, and bibliometric profile{---}and AI-use disclosure. Across six ERC-style scoring questions, we observe systematic score shifts relative to the blind condition when contextual information is introduced while proposal content is held fixed. The strongest penalties are associated with lower bibliometric profiles, followed by AI-use disclosure and lower-prestige institutions; name-based effects are smaller. These effects are consistent across prompts and evaluator models and are large enough to alter counterfactual rankings and top-$K$ selection outcomes. The results show that current LLM evaluators can be sensitive to contextual information attached to otherwise identical proposal content in ways that are fairness-relevant for research assessment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="rospocher-2026-contextual">
<titleInfo>
<title>Contextual Cue Sensitivity in LLM Evaluation of Grant Proposals: A Controlled ERC-Style Study</title>
</titleInfo>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Rospocher</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-440-8</identifier>
</relatedItem>
<abstract>Large language models (LLMs) are increasingly used as evaluators, but in high-stakes settings reliability requires more than consistency: scores should also be robust to contextual information that is not intended to alter the criterion being evaluated. We study this question in a controlled English-language ERC-style proposal-scoring setup. Starting from 40 Horizon Europe ERC-related project summaries, we construct synthetic Part B1-like proposal texts and evaluate them with eight LLMs under manipulations of principal-investigator (PI) profile information—name, institution, and bibliometric profile—and AI-use disclosure. Across six ERC-style scoring questions, we observe systematic score shifts relative to the blind condition when contextual information is introduced while proposal content is held fixed. The strongest penalties are associated with lower bibliometric profiles, followed by AI-use disclosure and lower-prestige institutions; name-based effects are smaller. These effects are consistent across prompts and evaluator models and are large enough to alter counterfactual rankings and top-K selection outcomes. The results show that current LLM evaluators can be sensitive to contextual information attached to otherwise identical proposal content in ways that are fairness-relevant for research assessment.</abstract>
<identifier type="citekey">rospocher-2026-contextual</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-short.44/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>679</start>
<end>698</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Contextual Cue Sensitivity in LLM Evaluation of Grant Proposals: A Controlled ERC-Style Study
%A Rospocher, Marco
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-440-8
%F rospocher-2026-contextual
%X Large language models (LLMs) are increasingly used as evaluators, but in high-stakes settings reliability requires more than consistency: scores should also be robust to contextual information that is not intended to alter the criterion being evaluated. We study this question in a controlled English-language ERC-style proposal-scoring setup. Starting from 40 Horizon Europe ERC-related project summaries, we construct synthetic Part B1-like proposal texts and evaluate them with eight LLMs under manipulations of principal-investigator (PI) profile information—name, institution, and bibliometric profile—and AI-use disclosure. Across six ERC-style scoring questions, we observe systematic score shifts relative to the blind condition when contextual information is introduced while proposal content is held fixed. The strongest penalties are associated with lower bibliometric profiles, followed by AI-use disclosure and lower-prestige institutions; name-based effects are smaller. These effects are consistent across prompts and evaluator models and are large enough to alter counterfactual rankings and top-K selection outcomes. The results show that current LLM evaluators can be sensitive to contextual information attached to otherwise identical proposal content in ways that are fairness-relevant for research assessment.
%U https://aclanthology.org/2026.aacl-short.44/
%P 679-698
Markdown (Informal)
[Contextual Cue Sensitivity in LLM Evaluation of Grant Proposals: A Controlled ERC-Style Study](https://aclanthology.org/2026.aacl-short.44/) (Rospocher, AACL-IJCNLP 2026)
ACL