@inproceedings{marchetti-etal-2026-calibrating,
title = "Calibrating {LLM}-as-a-Judge for Argument Quality Assessment through Prompting and Scoring Scale Design",
author = "Marchetti, Guilherme Aren and
Rocha, Gil and
Lopes Cardoso, Henrique",
editor = "Sousa-Silva, Rui and
Lopes Cardoso, Henrique and
Koponen, Maarit and
Pareja-Lora, Antonio",
booktitle = "Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era ({LUHME} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.luhme-1.16/",
pages = "197--214",
ISBN = "979-8-89176-471-2",
abstract = "Although LLM-as-a-Judge approaches have been successful in several domains, recent studies show that LLMs still struggle with Computational Argumentation tasks, particularly Argument Quality Assessment. Motivated by recent results in Information Retrieval showing that scoring granularity can affect the effectiveness of LLM-based document rankers, this paper investigates whether the same effect holds in Argument Quality Assessment scenarios.\par Our results demonstrate that, in several cases, more granular scoring ranges yield substantially higher agreement after mapping predictions back to the original scale. This contrasts with how humans evaluate, where reducing the granularity of possible scores can foster clearer, more consistent decisions. Additionally, our error analysis reveals a recurring limitation: LLMs tend to underestimate arguments labeled by human experts as high quality, frequently assigning them medium scores instead. This suggests that, despite the gains obtained through prompt and scale design, current LLMs remain poorly calibrated at the upper end of the argument quality scale."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="marchetti-etal-2026-calibrating">
<titleInfo>
<title>Calibrating LLM-as-a-Judge for Argument Quality Assessment through Prompting and Scoring Scale Design</title>
</titleInfo>
<name type="personal">
<namePart type="given">Guilherme</namePart>
<namePart type="given">Aren</namePart>
<namePart type="family">Marchetti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gil</namePart>
<namePart type="family">Rocha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henrique</namePart>
<namePart type="family">Lopes Cardoso</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era (LUHME 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Sousa-Silva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henrique</namePart>
<namePart type="family">Lopes Cardoso</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maarit</namePart>
<namePart type="family">Koponen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Pareja-Lora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-471-2</identifier>
</relatedItem>
<abstract>Although LLM-as-a-Judge approaches have been successful in several domains, recent studies show that LLMs still struggle with Computational Argumentation tasks, particularly Argument Quality Assessment. Motivated by recent results in Information Retrieval showing that scoring granularity can affect the effectiveness of LLM-based document rankers, this paper investigates whether the same effect holds in Argument Quality Assessment scenarios.\par Our results demonstrate that, in several cases, more granular scoring ranges yield substantially higher agreement after mapping predictions back to the original scale. This contrasts with how humans evaluate, where reducing the granularity of possible scores can foster clearer, more consistent decisions. Additionally, our error analysis reveals a recurring limitation: LLMs tend to underestimate arguments labeled by human experts as high quality, frequently assigning them medium scores instead. This suggests that, despite the gains obtained through prompt and scale design, current LLMs remain poorly calibrated at the upper end of the argument quality scale.</abstract>
<identifier type="citekey">marchetti-etal-2026-calibrating</identifier>
<location>
<url>https://aclanthology.org/2026.luhme-1.16/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>197</start>
<end>214</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Calibrating LLM-as-a-Judge for Argument Quality Assessment through Prompting and Scoring Scale Design
%A Marchetti, Guilherme Aren
%A Rocha, Gil
%A Lopes Cardoso, Henrique
%Y Sousa-Silva, Rui
%Y Lopes Cardoso, Henrique
%Y Koponen, Maarit
%Y Pareja-Lora, Antonio
%S Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era (LUHME 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-471-2
%F marchetti-etal-2026-calibrating
%X Although LLM-as-a-Judge approaches have been successful in several domains, recent studies show that LLMs still struggle with Computational Argumentation tasks, particularly Argument Quality Assessment. Motivated by recent results in Information Retrieval showing that scoring granularity can affect the effectiveness of LLM-based document rankers, this paper investigates whether the same effect holds in Argument Quality Assessment scenarios.\par Our results demonstrate that, in several cases, more granular scoring ranges yield substantially higher agreement after mapping predictions back to the original scale. This contrasts with how humans evaluate, where reducing the granularity of possible scores can foster clearer, more consistent decisions. Additionally, our error analysis reveals a recurring limitation: LLMs tend to underestimate arguments labeled by human experts as high quality, frequently assigning them medium scores instead. This suggests that, despite the gains obtained through prompt and scale design, current LLMs remain poorly calibrated at the upper end of the argument quality scale.
%U https://aclanthology.org/2026.luhme-1.16/
%P 197-214
Markdown (Informal)
[Calibrating LLM-as-a-Judge for Argument Quality Assessment through Prompting and Scoring Scale Design](https://aclanthology.org/2026.luhme-1.16/) (Marchetti et al., LUHME 2026)
ACL