@inproceedings{zgreaban-etal-2026-merge,
title = "{MERGE}: Minimal Expression-Replacement {GE}neralization Test for Natural Language Inference",
author = "Zgreab{\u{a}}n, M{\u{a}}d{\u{a}}lina and
Deoskar, Tejaswini and
Abzianidze, Lasha",
editor = "Arad, Dana and
Belinkov, Yonatan and
Chen, Hanjie and
Feldhus, Nils and
Mueller, Aaron and
Mohebbi, Hosein and
Sarti, Gabriele and
Tutek, Martin",
booktitle = "Proceedings of the 9th {B}lackbox{NLP} Workshop: Analyzing and Interpreting Neural Networks for {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.blackboxnlp-1.53/",
pages = "937--964",
ISBN = "979-8-89176-476-7",
abstract = "As many benchmarks have become saturated, it is increasingly important to create new datasets that evaluate the generalization capacity of current state-of-the-art models in reasoning. However, creating high-quality reasoning datasets is challenging: manual construction is costly, and automatic generation is error-prone, with the community therefore relying on synthetic datasets with limited scope. In this paper, we propose the Minimal Expression Replacement GEneralization (MERGE) test, to evaluate the robustness of reasoning models against minimal and non-adversarial variants of existing evaluation datasets. First, high-quality variants are automatically obtained from the original instances using Masked Language Models (MLMs) for generation together with safeguarding filters, called Minimal Expression Replacement (MERE). We then apply the MERGE test to Natural Language Inference (NLI), a popular reasoning task, by using MERE on two popular existing NLI datasets. We evaluate multiple strong NLI models and LLMs, and the results indicate they generalize poorly: both struggle to consistently and correctly classify variants minimally different in form, but similar in reasoning, from the original ones. We also analyze how aspects of variant generation, such as word class and source MLMs, affect model performance."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zgreaban-etal-2026-merge">
<titleInfo>
<title>MERGE: Minimal Expression-Replacement GEneralization Test for Natural Language Inference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Mădălina</namePart>
<namePart type="family">Zgreabăn</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tejaswini</namePart>
<namePart type="family">Deoskar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lasha</namePart>
<namePart type="family">Abzianidze</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dana</namePart>
<namePart type="family">Arad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yonatan</namePart>
<namePart type="family">Belinkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hanjie</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nils</namePart>
<namePart type="family">Feldhus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hosein</namePart>
<namePart type="family">Mohebbi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriele</namePart>
<namePart type="family">Sarti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Martin</namePart>
<namePart type="family">Tutek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-476-7</identifier>
</relatedItem>
<abstract>As many benchmarks have become saturated, it is increasingly important to create new datasets that evaluate the generalization capacity of current state-of-the-art models in reasoning. However, creating high-quality reasoning datasets is challenging: manual construction is costly, and automatic generation is error-prone, with the community therefore relying on synthetic datasets with limited scope. In this paper, we propose the Minimal Expression Replacement GEneralization (MERGE) test, to evaluate the robustness of reasoning models against minimal and non-adversarial variants of existing evaluation datasets. First, high-quality variants are automatically obtained from the original instances using Masked Language Models (MLMs) for generation together with safeguarding filters, called Minimal Expression Replacement (MERE). We then apply the MERGE test to Natural Language Inference (NLI), a popular reasoning task, by using MERE on two popular existing NLI datasets. We evaluate multiple strong NLI models and LLMs, and the results indicate they generalize poorly: both struggle to consistently and correctly classify variants minimally different in form, but similar in reasoning, from the original ones. We also analyze how aspects of variant generation, such as word class and source MLMs, affect model performance.</abstract>
<identifier type="citekey">zgreaban-etal-2026-merge</identifier>
<location>
<url>https://aclanthology.org/2026.blackboxnlp-1.53/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>937</start>
<end>964</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T MERGE: Minimal Expression-Replacement GEneralization Test for Natural Language Inference
%A Zgreabăn, Mădălina
%A Deoskar, Tejaswini
%A Abzianidze, Lasha
%Y Arad, Dana
%Y Belinkov, Yonatan
%Y Chen, Hanjie
%Y Feldhus, Nils
%Y Mueller, Aaron
%Y Mohebbi, Hosein
%Y Sarti, Gabriele
%Y Tutek, Martin
%S Proceedings of the 9th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-476-7
%F zgreaban-etal-2026-merge
%X As many benchmarks have become saturated, it is increasingly important to create new datasets that evaluate the generalization capacity of current state-of-the-art models in reasoning. However, creating high-quality reasoning datasets is challenging: manual construction is costly, and automatic generation is error-prone, with the community therefore relying on synthetic datasets with limited scope. In this paper, we propose the Minimal Expression Replacement GEneralization (MERGE) test, to evaluate the robustness of reasoning models against minimal and non-adversarial variants of existing evaluation datasets. First, high-quality variants are automatically obtained from the original instances using Masked Language Models (MLMs) for generation together with safeguarding filters, called Minimal Expression Replacement (MERE). We then apply the MERGE test to Natural Language Inference (NLI), a popular reasoning task, by using MERE on two popular existing NLI datasets. We evaluate multiple strong NLI models and LLMs, and the results indicate they generalize poorly: both struggle to consistently and correctly classify variants minimally different in form, but similar in reasoning, from the original ones. We also analyze how aspects of variant generation, such as word class and source MLMs, affect model performance.
%U https://aclanthology.org/2026.blackboxnlp-1.53/
%P 937-964
Markdown (Informal)
[MERGE: Minimal Expression-Replacement GEneralization Test for Natural Language Inference](https://aclanthology.org/2026.blackboxnlp-1.53/) (Zgreabăn et al., BlackboxNLP 2026)
ACL