@inproceedings{liang-etal-2026-mathdebugger,
title = "{M}ath{D}ebugger: A Benchmark for Type-Aware Error Detection in Synthetic Math Data",
author = "Liang, Hao and
Qiang, Meiyi and
Li, Yuying and
He, Zefeng and
Ma, Xiaochen and
An, Ruichuan and
Guo, Yongzhen and
Zhu, Zhengzhou and
Cui, Bin and
Zhang, Wentao",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.608/",
pages = "14001--14018",
ISBN = "979-8-89176-459-0",
abstract = "Synthetic math data is widely used to scale LLM training, but its correctness is not guaranteed. We introduce MathDebugger, the first text-only, type-aware benchmark for jointly detecting *and* classifying errors in synthetic mathematical questions and answers. MathDebugger contains 2,000 correct questions, 2,000 erroneous questions (balanced across four fine-grained error types), and 2,000 annotated answers (610 erroneous, across three error types), each annotated with both a correctness label and an error-type label, with per-error-type Fleiss' $\kappa$ ranging from 0.69 to 0.88. On 14 SOTA LLMs (6 closed-source, 8 open-source, 1.5B{--}72B) and 3 process reward models, even advanced systems such as GPT-o3 and DeepSeek-R1 remain far from saturating the benchmark, particularly on fine-grained error-type detection; we further surface a *solving-vs-verifying* gap that causes math-tuned and long-chain-reasoning models to trail their general-purpose siblings. Error-type annotations also provide actionable supervision for correction, with gains that are significant under paired bootstrap ($p<0.01$). Code and data: [https://anonymous.4open.science/r/MathDebugger-7C4A/README.md](https://anonymous.4open.science/r/MathDebugger-7C4A/README.md)"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liang-etal-2026-mathdebugger">
<titleInfo>
<title>MathDebugger: A Benchmark for Type-Aware Error Detection in Synthetic Math Data</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Liang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Meiyi</namePart>
<namePart type="family">Qiang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yuying</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zefeng</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiaochen</namePart>
<namePart type="family">Ma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruichuan</namePart>
<namePart type="family">An</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongzhen</namePart>
<namePart type="family">Guo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhengzhou</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bin</namePart>
<namePart type="family">Cui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wentao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Synthetic math data is widely used to scale LLM training, but its correctness is not guaranteed. We introduce MathDebugger, the first text-only, type-aware benchmark for jointly detecting *and* classifying errors in synthetic mathematical questions and answers. MathDebugger contains 2,000 correct questions, 2,000 erroneous questions (balanced across four fine-grained error types), and 2,000 annotated answers (610 erroneous, across three error types), each annotated with both a correctness label and an error-type label, with per-error-type Fleiss’ ąppa ranging from 0.69 to 0.88. On 14 SOTA LLMs (6 closed-source, 8 open-source, 1.5B–72B) and 3 process reward models, even advanced systems such as GPT-o3 and DeepSeek-R1 remain far from saturating the benchmark, particularly on fine-grained error-type detection; we further surface a *solving-vs-verifying* gap that causes math-tuned and long-chain-reasoning models to trail their general-purpose siblings. Error-type annotations also provide actionable supervision for correction, with gains that are significant under paired bootstrap (p<0.01). Code and data: [https://anonymous.4open.science/r/MathDebugger-7C4A/README.md](https://anonymous.4open.science/r/MathDebugger-7C4A/README.md)</abstract>
<identifier type="citekey">liang-etal-2026-mathdebugger</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.608/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>14001</start>
<end>14018</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MathDebugger: A Benchmark for Type-Aware Error Detection in Synthetic Math Data
%A Liang, Hao
%A Qiang, Meiyi
%A Li, Yuying
%A He, Zefeng
%A Ma, Xiaochen
%A An, Ruichuan
%A Guo, Yongzhen
%A Zhu, Zhengzhou
%A Cui, Bin
%A Zhang, Wentao
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F liang-etal-2026-mathdebugger
%X Synthetic math data is widely used to scale LLM training, but its correctness is not guaranteed. We introduce MathDebugger, the first text-only, type-aware benchmark for jointly detecting *and* classifying errors in synthetic mathematical questions and answers. MathDebugger contains 2,000 correct questions, 2,000 erroneous questions (balanced across four fine-grained error types), and 2,000 annotated answers (610 erroneous, across three error types), each annotated with both a correctness label and an error-type label, with per-error-type Fleiss’ ąppa ranging from 0.69 to 0.88. On 14 SOTA LLMs (6 closed-source, 8 open-source, 1.5B–72B) and 3 process reward models, even advanced systems such as GPT-o3 and DeepSeek-R1 remain far from saturating the benchmark, particularly on fine-grained error-type detection; we further surface a *solving-vs-verifying* gap that causes math-tuned and long-chain-reasoning models to trail their general-purpose siblings. Error-type annotations also provide actionable supervision for correction, with gains that are significant under paired bootstrap (p<0.01). Code and data: [https://anonymous.4open.science/r/MathDebugger-7C4A/README.md](https://anonymous.4open.science/r/MathDebugger-7C4A/README.md)
%U https://aclanthology.org/2026.emnlp-main.608/
%P 14001-14018
Markdown (Informal)
[MathDebugger: A Benchmark for Type-Aware Error Detection in Synthetic Math Data](https://aclanthology.org/2026.emnlp-main.608/) (Liang et al., EMNLP 2026)
ACL
- Hao Liang, Meiyi Qiang, Yuying Li, Zefeng He, Xiaochen Ma, Ruichuan An, Yongzhen Guo, Zhengzhou Zhu, Bin Cui, and Wentao Zhang. 2026. MathDebugger: A Benchmark for Type-Aware Error Detection in Synthetic Math Data. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 14001–14018, Budapest, Hungary. Association for Computational Linguistics.