@inproceedings{dias-alexiou-etal-2026-replacing,
title = "Replacing the Human in the Loop: Towards Automated Evaluation of {LLM} Mathematical Proof Generalization",
author = "Dias-Alexiou, Carolina and
Marrese-Taylor, Edison and
Solano, Alfredo and
Matsuo, Yutaka",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.50/",
pages = "996--1004",
ISBN = "979-8-89176-444-6",
abstract = "As we continue to extend their range of applications, the study of the generalization capabilities of large language models (LLM) has become an important topic. Previous works have recently proposed to study these abilities in the context of mathematical reasoning. This was done by asking models to reproduce proofs that they have most likely seen during training, but when key symbols are replaced. Since this evaluation relies heavily on human feedback, we study to what degree this evaluation can be automatically performed. To that end, we simulate the human evaluation proposed by previous work by interactively presenting the same questions to the LLMs as prompts. We analyze the degree of alignment between human and model answers via an accuracy and correlation analysis, as well as via inter-annotator agreement metrics. We test a selection of models, including ones with reasoning capabilities, and show LLMs currently have important limitations in this regard, which cannot be addressed by ICL."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="dias-alexiou-etal-2026-replacing">
<titleInfo>
<title>Replacing the Human in the Loop: Towards Automated Evaluation of LLM Mathematical Proof Generalization</title>
</titleInfo>
<name type="personal">
<namePart type="given">Carolina</namePart>
<namePart type="family">Dias-Alexiou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Edison</namePart>
<namePart type="family">Marrese-Taylor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alfredo</namePart>
<namePart type="family">Solano</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yutaka</namePart>
<namePart type="family">Matsuo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>As we continue to extend their range of applications, the study of the generalization capabilities of large language models (LLM) has become an important topic. Previous works have recently proposed to study these abilities in the context of mathematical reasoning. This was done by asking models to reproduce proofs that they have most likely seen during training, but when key symbols are replaced. Since this evaluation relies heavily on human feedback, we study to what degree this evaluation can be automatically performed. To that end, we simulate the human evaluation proposed by previous work by interactively presenting the same questions to the LLMs as prompts. We analyze the degree of alignment between human and model answers via an accuracy and correlation analysis, as well as via inter-annotator agreement metrics. We test a selection of models, including ones with reasoning capabilities, and show LLMs currently have important limitations in this regard, which cannot be addressed by ICL.</abstract>
<identifier type="citekey">dias-alexiou-etal-2026-replacing</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.50/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>996</start>
<end>1004</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Replacing the Human in the Loop: Towards Automated Evaluation of LLM Mathematical Proof Generalization
%A Dias-Alexiou, Carolina
%A Marrese-Taylor, Edison
%A Solano, Alfredo
%A Matsuo, Yutaka
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F dias-alexiou-etal-2026-replacing
%X As we continue to extend their range of applications, the study of the generalization capabilities of large language models (LLM) has become an important topic. Previous works have recently proposed to study these abilities in the context of mathematical reasoning. This was done by asking models to reproduce proofs that they have most likely seen during training, but when key symbols are replaced. Since this evaluation relies heavily on human feedback, we study to what degree this evaluation can be automatically performed. To that end, we simulate the human evaluation proposed by previous work by interactively presenting the same questions to the LLMs as prompts. We analyze the degree of alignment between human and model answers via an accuracy and correlation analysis, as well as via inter-annotator agreement metrics. We test a selection of models, including ones with reasoning capabilities, and show LLMs currently have important limitations in this regard, which cannot be addressed by ICL.
%U https://aclanthology.org/2026.findings-aacl.50/
%P 996-1004
Markdown (Informal)
[Replacing the Human in the Loop: Towards Automated Evaluation of LLM Mathematical Proof Generalization](https://aclanthology.org/2026.findings-aacl.50/) (Dias-Alexiou et al., Findings 2026)
ACL