@inproceedings{benigeri-birnbaum-2026-characterizing,
title = "Characterizing and assessing robust conversational interaction: Amendments in order specification",
author = "Benigeri, Simon and
Birnbaum, Larry",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1380/",
doi = "10.18653/2026.findings-emnlp.1380",
pages = "29321--29360",
ISBN = "979-8-89176-462-0",
abstract = "We assess the robustness of large language models (LLMs) on a simple task{---}fast-food order specification{---}in the face of user-initiated amendments. In amending an order, users must communicate, explicitly or implicitly, that a change is being made, the type of change, and what is being changed. We generate synthetic food orders with single amendments, controlling the type of amendment, which components are implicitly or incompletely specified, and the pragmatic inferences required to resolve them. On a dataset broadly covering the space of possible combinations, 7 flagship models average 97.7{\%} accuracy. On a more targeted dataset with denser coverage of empirically difficult patterns, accuracy drops to 93.4{\%}, with the hardest combinations dropping below 80{\%} accuracy. We also test two further prompting methods: generating an order before interpreting amendments in a turn-taking interaction; and function-calling to parameterize explicit edit operations on the order itself. These somewhat improve performance (+0.2 pp and +3.2 pp respectively), but neither entirely eliminates the failure modes. We also find that asking clarification questions about implicit components improves accuracy on a subset of difficult examples (+5.4 pp), but models ask on only 11{\%} of examples. Given the task{'}s deliberate simplicity{---}small fixed menu, a single standard order, with a single unambiguous amendment{---}these results illustrate the challenge of robustly performing the pragmatic inferences that real-world conversational interaction demands."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="benigeri-birnbaum-2026-characterizing">
<titleInfo>
<title>Characterizing and assessing robust conversational interaction: Amendments in order specification</title>
</titleInfo>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Benigeri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Larry</namePart>
<namePart type="family">Birnbaum</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We assess the robustness of large language models (LLMs) on a simple task—fast-food order specification—in the face of user-initiated amendments. In amending an order, users must communicate, explicitly or implicitly, that a change is being made, the type of change, and what is being changed. We generate synthetic food orders with single amendments, controlling the type of amendment, which components are implicitly or incompletely specified, and the pragmatic inferences required to resolve them. On a dataset broadly covering the space of possible combinations, 7 flagship models average 97.7% accuracy. On a more targeted dataset with denser coverage of empirically difficult patterns, accuracy drops to 93.4%, with the hardest combinations dropping below 80% accuracy. We also test two further prompting methods: generating an order before interpreting amendments in a turn-taking interaction; and function-calling to parameterize explicit edit operations on the order itself. These somewhat improve performance (+0.2 pp and +3.2 pp respectively), but neither entirely eliminates the failure modes. We also find that asking clarification questions about implicit components improves accuracy on a subset of difficult examples (+5.4 pp), but models ask on only 11% of examples. Given the task’s deliberate simplicity—small fixed menu, a single standard order, with a single unambiguous amendment—these results illustrate the challenge of robustly performing the pragmatic inferences that real-world conversational interaction demands.</abstract>
<identifier type="citekey">benigeri-birnbaum-2026-characterizing</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1380</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1380/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>29321</start>
<end>29360</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Characterizing and assessing robust conversational interaction: Amendments in order specification
%A Benigeri, Simon
%A Birnbaum, Larry
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F benigeri-birnbaum-2026-characterizing
%X We assess the robustness of large language models (LLMs) on a simple task—fast-food order specification—in the face of user-initiated amendments. In amending an order, users must communicate, explicitly or implicitly, that a change is being made, the type of change, and what is being changed. We generate synthetic food orders with single amendments, controlling the type of amendment, which components are implicitly or incompletely specified, and the pragmatic inferences required to resolve them. On a dataset broadly covering the space of possible combinations, 7 flagship models average 97.7% accuracy. On a more targeted dataset with denser coverage of empirically difficult patterns, accuracy drops to 93.4%, with the hardest combinations dropping below 80% accuracy. We also test two further prompting methods: generating an order before interpreting amendments in a turn-taking interaction; and function-calling to parameterize explicit edit operations on the order itself. These somewhat improve performance (+0.2 pp and +3.2 pp respectively), but neither entirely eliminates the failure modes. We also find that asking clarification questions about implicit components improves accuracy on a subset of difficult examples (+5.4 pp), but models ask on only 11% of examples. Given the task’s deliberate simplicity—small fixed menu, a single standard order, with a single unambiguous amendment—these results illustrate the challenge of robustly performing the pragmatic inferences that real-world conversational interaction demands.
%R 10.18653/2026.findings-emnlp.1380
%U https://aclanthology.org/2026.findings-emnlp.1380/
%U https://doi.org/10.18653/2026.findings-emnlp.1380
%P 29321-29360
Markdown (Informal)
[Characterizing and assessing robust conversational interaction: Amendments in order specification](https://aclanthology.org/2026.findings-emnlp.1380/) (Benigeri & Birnbaum, Findings 2026)
ACL