@inproceedings{joshi-etal-2026-whats,
title = "What{'}s Worth Asking? Evaluating Preference Elicitation in Transactional Agents",
author = "Joshi, Kandarp and
Mittal, Tushar and
Soni, Chirag and
Zhou, Lillian and
Aleksic, Petar",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.174/",
pages = "3888--3914",
ISBN = "979-8-89176-439-2",
abstract = "Transactional agents must act on underspecified requests. ``Book me a flight from Chicago to San Francisco'' omits budget, timing, stop tolerance, and airline loyalty, any of which can change the best option by hundreds of dollars. The challenge is identifying, without a user-specific prior, which unstated attribute preference is most likely to change the recommendation.\par We introduce ELICITBENCH, a benchmark of 1,387 travel and housing scenarios pairing real search-result pages (SERPs) with transaction-grounded preference profiles. Each scenario partitions attributes into stated and hidden preferences. An ex-post analytical oracle assigns question-value labels based on the normalized reduction in market-conditioned recommendation regret. Agents are evaluated by task success, question count, and net utility under a super-linear patience cost.\par ELICITBENCH reveals that plausible questions are often not decision-relevant: on Gemini 2.5 Flash and Claude Sonnet 4.5, unguided agents ask natural follow-up questions with little downstream value. SERP-aware prompting raises net utility from 0.073 to 0.306 and task success from 21.7{\%} to 57.6{\%} on Gemini, and net utility from 0.221 to 0.346 on Sonnet while asking fewer questions. The selection of items is a second bottleneck: with every preference stated, agents still pick the wrong item in about 11{\%} of the scenarios."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="joshi-etal-2026-whats">
<titleInfo>
<title>What’s Worth Asking? Evaluating Preference Elicitation in Transactional Agents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kandarp</namePart>
<namePart type="family">Joshi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tushar</namePart>
<namePart type="family">Mittal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chirag</namePart>
<namePart type="family">Soni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lillian</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Petar</namePart>
<namePart type="family">Aleksic</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Transactional agents must act on underspecified requests. “Book me a flight from Chicago to San Francisco” omits budget, timing, stop tolerance, and airline loyalty, any of which can change the best option by hundreds of dollars. The challenge is identifying, without a user-specific prior, which unstated attribute preference is most likely to change the recommendation.\par We introduce ELICITBENCH, a benchmark of 1,387 travel and housing scenarios pairing real search-result pages (SERPs) with transaction-grounded preference profiles. Each scenario partitions attributes into stated and hidden preferences. An ex-post analytical oracle assigns question-value labels based on the normalized reduction in market-conditioned recommendation regret. Agents are evaluated by task success, question count, and net utility under a super-linear patience cost.\par ELICITBENCH reveals that plausible questions are often not decision-relevant: on Gemini 2.5 Flash and Claude Sonnet 4.5, unguided agents ask natural follow-up questions with little downstream value. SERP-aware prompting raises net utility from 0.073 to 0.306 and task success from 21.7% to 57.6% on Gemini, and net utility from 0.221 to 0.346 on Sonnet while asking fewer questions. The selection of items is a second bottleneck: with every preference stated, agents still pick the wrong item in about 11% of the scenarios.</abstract>
<identifier type="citekey">joshi-etal-2026-whats</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.174/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3888</start>
<end>3914</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What’s Worth Asking? Evaluating Preference Elicitation in Transactional Agents
%A Joshi, Kandarp
%A Mittal, Tushar
%A Soni, Chirag
%A Zhou, Lillian
%A Aleksic, Petar
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F joshi-etal-2026-whats
%X Transactional agents must act on underspecified requests. “Book me a flight from Chicago to San Francisco” omits budget, timing, stop tolerance, and airline loyalty, any of which can change the best option by hundreds of dollars. The challenge is identifying, without a user-specific prior, which unstated attribute preference is most likely to change the recommendation.\par We introduce ELICITBENCH, a benchmark of 1,387 travel and housing scenarios pairing real search-result pages (SERPs) with transaction-grounded preference profiles. Each scenario partitions attributes into stated and hidden preferences. An ex-post analytical oracle assigns question-value labels based on the normalized reduction in market-conditioned recommendation regret. Agents are evaluated by task success, question count, and net utility under a super-linear patience cost.\par ELICITBENCH reveals that plausible questions are often not decision-relevant: on Gemini 2.5 Flash and Claude Sonnet 4.5, unguided agents ask natural follow-up questions with little downstream value. SERP-aware prompting raises net utility from 0.073 to 0.306 and task success from 21.7% to 57.6% on Gemini, and net utility from 0.221 to 0.346 on Sonnet while asking fewer questions. The selection of items is a second bottleneck: with every preference stated, agents still pick the wrong item in about 11% of the scenarios.
%U https://aclanthology.org/2026.aacl-long.174/
%P 3888-3914
Markdown (Informal)
[What’s Worth Asking? Evaluating Preference Elicitation in Transactional Agents](https://aclanthology.org/2026.aacl-long.174/) (Joshi et al., AACL-IJCNLP 2026)
ACL
- Kandarp Joshi, Tushar Mittal, Chirag Soni, Lillian Zhou, and Petar Aleksic. 2026. What’s Worth Asking? Evaluating Preference Elicitation in Transactional Agents. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 3888–3914, Hengqin, China. Association for Computational Linguistics.