@inproceedings{ocampo-etal-2026-argument,
title = "Argument Quality Assessment with Large Language Models: A Pairwise Bradley-Terry Approach",
author = "Ocampo, Nicol{\'a}s Benjam{\'i}n and
Nyiranziza, Agnes Paullate and
Ceolin, Davide",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1175/",
doi = "10.18653/2026.findings-emnlp.1175",
pages = "24842--24857",
ISBN = "979-8-89176-462-0",
abstract = "Large Language Models (LLMs) have demonstrated remarkable capabilities in tasks related to reasoning and judgment. However, assessing the quality of arguments requires a rigorous evaluation. We investigate the extent to which LLMs can effectively perform this task. We tested 12 open-weight LLMs of different sizes and families under zero-shot, few-shot, and chain-of-thought to approximate human pairwise comparisons of argument quality across three dimensions{---}logical, rhetorical, and dialectic{---}and used these comparisons in a Bradley-Terry model to infer latent strength scores and derive a ranking of arguments. Our insights show that LLMs have promising but moderate correlation with human judgment, with Llama-70B obtaining the strongest alignment, reaching moderate Cohen{'}s {\ensuremath{\kappa}} = 0.493 and moderate correlations with Bradley-Terry scores derived from these annotations (Kendall, Pearson, and Spearman: 0.327-0.477). Other LLMs exhibit weak, moderate, or high alignment with Llama-70B while achieving comparable results against human judgment, suggesting partial but complementary understanding of underlying quality dimensions despite differences in model size and family. Moreover, LLM predictions are stable across trial runs, with fewer than 7.75{\%} of cases yielding different labels. Remaining variability is handled via majority voting and few-shot prompting for large-size models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ocampo-etal-2026-argument">
<titleInfo>
<title>Argument Quality Assessment with Large Language Models: A Pairwise Bradley-Terry Approach</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nicolás</namePart>
<namePart type="given">Benjamín</namePart>
<namePart type="family">Ocampo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Agnes</namePart>
<namePart type="given">Paullate</namePart>
<namePart type="family">Nyiranziza</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Davide</namePart>
<namePart type="family">Ceolin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) have demonstrated remarkable capabilities in tasks related to reasoning and judgment. However, assessing the quality of arguments requires a rigorous evaluation. We investigate the extent to which LLMs can effectively perform this task. We tested 12 open-weight LLMs of different sizes and families under zero-shot, few-shot, and chain-of-thought to approximate human pairwise comparisons of argument quality across three dimensions—logical, rhetorical, and dialectic—and used these comparisons in a Bradley-Terry model to infer latent strength scores and derive a ranking of arguments. Our insights show that LLMs have promising but moderate correlation with human judgment, with Llama-70B obtaining the strongest alignment, reaching moderate Cohen’s \ensuremathąppa = 0.493 and moderate correlations with Bradley-Terry scores derived from these annotations (Kendall, Pearson, and Spearman: 0.327-0.477). Other LLMs exhibit weak, moderate, or high alignment with Llama-70B while achieving comparable results against human judgment, suggesting partial but complementary understanding of underlying quality dimensions despite differences in model size and family. Moreover, LLM predictions are stable across trial runs, with fewer than 7.75% of cases yielding different labels. Remaining variability is handled via majority voting and few-shot prompting for large-size models.</abstract>
<identifier type="citekey">ocampo-etal-2026-argument</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1175</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1175/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24842</start>
<end>24857</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Argument Quality Assessment with Large Language Models: A Pairwise Bradley-Terry Approach
%A Ocampo, Nicolás Benjamín
%A Nyiranziza, Agnes Paullate
%A Ceolin, Davide
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ocampo-etal-2026-argument
%X Large Language Models (LLMs) have demonstrated remarkable capabilities in tasks related to reasoning and judgment. However, assessing the quality of arguments requires a rigorous evaluation. We investigate the extent to which LLMs can effectively perform this task. We tested 12 open-weight LLMs of different sizes and families under zero-shot, few-shot, and chain-of-thought to approximate human pairwise comparisons of argument quality across three dimensions—logical, rhetorical, and dialectic—and used these comparisons in a Bradley-Terry model to infer latent strength scores and derive a ranking of arguments. Our insights show that LLMs have promising but moderate correlation with human judgment, with Llama-70B obtaining the strongest alignment, reaching moderate Cohen’s \ensuremathąppa = 0.493 and moderate correlations with Bradley-Terry scores derived from these annotations (Kendall, Pearson, and Spearman: 0.327-0.477). Other LLMs exhibit weak, moderate, or high alignment with Llama-70B while achieving comparable results against human judgment, suggesting partial but complementary understanding of underlying quality dimensions despite differences in model size and family. Moreover, LLM predictions are stable across trial runs, with fewer than 7.75% of cases yielding different labels. Remaining variability is handled via majority voting and few-shot prompting for large-size models.
%R 10.18653/2026.findings-emnlp.1175
%U https://aclanthology.org/2026.findings-emnlp.1175/
%U https://doi.org/10.18653/2026.findings-emnlp.1175
%P 24842-24857
Markdown (Informal)
[Argument Quality Assessment with Large Language Models: A Pairwise Bradley-Terry Approach](https://aclanthology.org/2026.findings-emnlp.1175/) (Ocampo et al., Findings 2026)
ACL