@inproceedings{belluzzo-etal-2026-house,
title = "This House Debates {AI}: Evaluating a Language Model in {O}xford-Style Debates against Human Experts",
author = {Belluzzo, Umberto and
Hackenburg, Kobi and
Kirk, Hannah Rose and
Hale, Scott and
R{\"o}ttger, Paul},
editor = "Piperidis, Stelios and
Bel, N{\'u}ria and
van den Heuvel, Henk and
Ide, Nancy and
Krek, Simon and
Toral, Antonio",
booktitle = "Proceedings of the Fifteenth Language Resources and Evaluation Conference",
month = may,
year = "2026",
address = "Palma de Mallorca, Spain",
publisher = "ELRA Language Resource Association",
url = "https://aclanthology.org/2026.lrec-1.215/",
doi = "10.63317/3ep9kdi62wpr",
pages = "2742--2759",
abstract = "Recent work shows that large language models (LLMs) are increasingly capable of generating persuasive arguments and messages, creating concerns over undue influence on human beliefs. Most evidence so far, however, evaluates LLM argumentation and persuasion in single-turn interactions and/or compares to weak human baselines. To address this gap, we benchmark a state-of-the-art LLM, Llama 3.1 Instruct 405B, in 100 six-turn Oxford-style debates against 20 experienced human debaters. Each anonymised debate is rated by 5 independent raters, who provide win/loss judgments as well as 0{--}100 scores across 11 dimensions of quality. Based on these ratings, the LLM is competitive overall, with a win rate of 51.2{\%}, ranking 6th out of 21 debaters on mean performance score. Compared to humans, the LLM generally scores higher on presentational dimensions (e.g., clarity, confidence, formality) but equal on most substantive dimensions (convincingness, evidence, originality). We also find that pre/post rater stance tends to shift towards the position raters chose as the winning side, regardless of whether this side was the LLM or a human. Overall, our results provide new evidence on the qualities of LLM argumentation and its drivers, suggesting strong argumentative competence even in competitive multi-turn settings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="belluzzo-etal-2026-house">
<titleInfo>
<title>This House Debates AI: Evaluating a Language Model in Oxford-Style Debates against Human Experts</title>
</titleInfo>
<name type="personal">
<namePart type="given">Umberto</namePart>
<namePart type="family">Belluzzo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kobi</namePart>
<namePart type="family">Hackenburg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hannah</namePart>
<namePart type="given">Rose</namePart>
<namePart type="family">Kirk</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Scott</namePart>
<namePart type="family">Hale</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paul</namePart>
<namePart type="family">Röttger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifteenth Language Resources and Evaluation Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Stelios</namePart>
<namePart type="family">Piperidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Núria</namePart>
<namePart type="family">Bel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henk</namePart>
<namePart type="family">van den Heuvel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nancy</namePart>
<namePart type="family">Ide</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Krek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Toral</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resource Association</publisher>
<place>
<placeTerm type="text">Palma de Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Recent work shows that large language models (LLMs) are increasingly capable of generating persuasive arguments and messages, creating concerns over undue influence on human beliefs. Most evidence so far, however, evaluates LLM argumentation and persuasion in single-turn interactions and/or compares to weak human baselines. To address this gap, we benchmark a state-of-the-art LLM, Llama 3.1 Instruct 405B, in 100 six-turn Oxford-style debates against 20 experienced human debaters. Each anonymised debate is rated by 5 independent raters, who provide win/loss judgments as well as 0–100 scores across 11 dimensions of quality. Based on these ratings, the LLM is competitive overall, with a win rate of 51.2%, ranking 6th out of 21 debaters on mean performance score. Compared to humans, the LLM generally scores higher on presentational dimensions (e.g., clarity, confidence, formality) but equal on most substantive dimensions (convincingness, evidence, originality). We also find that pre/post rater stance tends to shift towards the position raters chose as the winning side, regardless of whether this side was the LLM or a human. Overall, our results provide new evidence on the qualities of LLM argumentation and its drivers, suggesting strong argumentative competence even in competitive multi-turn settings.</abstract>
<identifier type="citekey">belluzzo-etal-2026-house</identifier>
<identifier type="doi">10.63317/3ep9kdi62wpr</identifier>
<location>
<url>https://aclanthology.org/2026.lrec-1.215/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>2742</start>
<end>2759</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T This House Debates AI: Evaluating a Language Model in Oxford-Style Debates against Human Experts
%A Belluzzo, Umberto
%A Hackenburg, Kobi
%A Kirk, Hannah Rose
%A Hale, Scott
%A Röttger, Paul
%Y Piperidis, Stelios
%Y Bel, Núria
%Y van den Heuvel, Henk
%Y Ide, Nancy
%Y Krek, Simon
%Y Toral, Antonio
%S Proceedings of the Fifteenth Language Resources and Evaluation Conference
%D 2026
%8 May
%I ELRA Language Resource Association
%C Palma de Mallorca, Spain
%F belluzzo-etal-2026-house
%X Recent work shows that large language models (LLMs) are increasingly capable of generating persuasive arguments and messages, creating concerns over undue influence on human beliefs. Most evidence so far, however, evaluates LLM argumentation and persuasion in single-turn interactions and/or compares to weak human baselines. To address this gap, we benchmark a state-of-the-art LLM, Llama 3.1 Instruct 405B, in 100 six-turn Oxford-style debates against 20 experienced human debaters. Each anonymised debate is rated by 5 independent raters, who provide win/loss judgments as well as 0–100 scores across 11 dimensions of quality. Based on these ratings, the LLM is competitive overall, with a win rate of 51.2%, ranking 6th out of 21 debaters on mean performance score. Compared to humans, the LLM generally scores higher on presentational dimensions (e.g., clarity, confidence, formality) but equal on most substantive dimensions (convincingness, evidence, originality). We also find that pre/post rater stance tends to shift towards the position raters chose as the winning side, regardless of whether this side was the LLM or a human. Overall, our results provide new evidence on the qualities of LLM argumentation and its drivers, suggesting strong argumentative competence even in competitive multi-turn settings.
%R 10.63317/3ep9kdi62wpr
%U https://aclanthology.org/2026.lrec-1.215/
%U https://doi.org/10.63317/3ep9kdi62wpr
%P 2742-2759
Markdown (Informal)
[This House Debates AI: Evaluating a Language Model in Oxford-Style Debates against Human Experts](https://aclanthology.org/2026.lrec-1.215/) (Belluzzo et al., LREC 2026)
ACL