@inproceedings{delgado-2026-score,
title = "Do Not Score the Edited Answer: A Negative Result on Loose Evaluation for Verifiable Instruction Following",
author = "Delgado, Jos{\'e} Luis",
editor = "Sedoc, Jo{\~a}o and
Shu, Raphael and
Drozd, Aleksandr and
Akula, Arjun and
Rogers, Anna and
Giorgi, Salvatore and
Benton, Adrian",
booktitle = "Proceedings of the Seventh Workshop on Insights from Negative Results in {NLP}",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.insights-1.15/",
pages = "115--119",
ISBN = "979-8-89176-480-4",
abstract = "Verifiable instruction-following benchmarks often include strict and loose scoring modes: strict scoring checks the original response, and loose scoring applies the checker to transformed versions of the response, for example after removing markdown or dropping boundary lines, to recover false negatives caused by harmless wrappers and formatting artifacts. We study a negative result for this repair strategy using a constructive suite of 144 canaries in which every valid wrapper is paired with an invalid deletion trap that has the same transformed-response checker outcomes under IFEval-style transformations. Loose scoring accepts all valid wrappers and all invalid traps; support over transformation functions accepts both classes, and a threshold that deduplicates passing edited strings rejects both classes. On the balanced suite, any scorer that observes only the transformed-view pass signature has a best possible accuracy of 0.50, and the canaries are solved only by a boundary-aware certificate that inspects the original deleted span. Deletion- or normalization-based loose evaluation therefore needs evidence-preservation checks when edited spans may affect instruction satisfaction."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="delgado-2026-score">
<titleInfo>
<title>Do Not Score the Edited Answer: A Negative Result on Loose Evaluation for Verifiable Instruction Following</title>
</titleInfo>
<name type="personal">
<namePart type="given">José</namePart>
<namePart type="given">Luis</namePart>
<namePart type="family">Delgado</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Seventh Workshop on Insights from Negative Results in NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">João</namePart>
<namePart type="family">Sedoc</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raphael</namePart>
<namePart type="family">Shu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aleksandr</namePart>
<namePart type="family">Drozd</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arjun</namePart>
<namePart type="family">Akula</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Rogers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salvatore</namePart>
<namePart type="family">Giorgi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Adrian</namePart>
<namePart type="family">Benton</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-480-4</identifier>
</relatedItem>
<abstract>Verifiable instruction-following benchmarks often include strict and loose scoring modes: strict scoring checks the original response, and loose scoring applies the checker to transformed versions of the response, for example after removing markdown or dropping boundary lines, to recover false negatives caused by harmless wrappers and formatting artifacts. We study a negative result for this repair strategy using a constructive suite of 144 canaries in which every valid wrapper is paired with an invalid deletion trap that has the same transformed-response checker outcomes under IFEval-style transformations. Loose scoring accepts all valid wrappers and all invalid traps; support over transformation functions accepts both classes, and a threshold that deduplicates passing edited strings rejects both classes. On the balanced suite, any scorer that observes only the transformed-view pass signature has a best possible accuracy of 0.50, and the canaries are solved only by a boundary-aware certificate that inspects the original deleted span. Deletion- or normalization-based loose evaluation therefore needs evidence-preservation checks when edited spans may affect instruction satisfaction.</abstract>
<identifier type="citekey">delgado-2026-score</identifier>
<location>
<url>https://aclanthology.org/2026.insights-1.15/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>115</start>
<end>119</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Do Not Score the Edited Answer: A Negative Result on Loose Evaluation for Verifiable Instruction Following
%A Delgado, José Luis
%Y Sedoc, João
%Y Shu, Raphael
%Y Drozd, Aleksandr
%Y Akula, Arjun
%Y Rogers, Anna
%Y Giorgi, Salvatore
%Y Benton, Adrian
%S Proceedings of the Seventh Workshop on Insights from Negative Results in NLP
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-480-4
%F delgado-2026-score
%X Verifiable instruction-following benchmarks often include strict and loose scoring modes: strict scoring checks the original response, and loose scoring applies the checker to transformed versions of the response, for example after removing markdown or dropping boundary lines, to recover false negatives caused by harmless wrappers and formatting artifacts. We study a negative result for this repair strategy using a constructive suite of 144 canaries in which every valid wrapper is paired with an invalid deletion trap that has the same transformed-response checker outcomes under IFEval-style transformations. Loose scoring accepts all valid wrappers and all invalid traps; support over transformation functions accepts both classes, and a threshold that deduplicates passing edited strings rejects both classes. On the balanced suite, any scorer that observes only the transformed-view pass signature has a best possible accuracy of 0.50, and the canaries are solved only by a boundary-aware certificate that inspects the original deleted span. Deletion- or normalization-based loose evaluation therefore needs evidence-preservation checks when edited spans may affect instruction satisfaction.
%U https://aclanthology.org/2026.insights-1.15/
%P 115-119
Markdown (Informal)
[Do Not Score the Edited Answer: A Negative Result on Loose Evaluation for Verifiable Instruction Following](https://aclanthology.org/2026.insights-1.15/) (Delgado, insights 2026)
ACL