@inproceedings{kim-etal-2026-igg,
title = "{IGG}: A Benchmark for Interactive {GUI} Grounding under Visibility Constraints",
author = "Kim, Kyeong Seon and
Son, Jiyeon and
Oh, Tae-Hyun",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.2703/",
pages = "61475--61498",
ISBN = "979-8-89176-459-0",
abstract = "GUI grounding benchmarks ask whether agents can localize a target that is already visible in a static screenshot. Real interfaces often violate this assumption: a target may be off-screen, appear only after a state change or hover, remain occluded, or become actionable only after a delay. This leaves the ability to recover target visibility unevaluated. To measure this reveal step, we introduce Interactive GUI Grounding (IGG), a benchmark in which an agent must first make the target visible through interaction and then click it. IGG contains 601 tasks across 16 controllable GUI environments and seven visibility-constraint sub-types, with a compact action space of scrolling, moving, dragging, waiting, and clicking. In our human study, participants achieve 94.6{\%} accuracy, while GPT-5.5 reaches 79.4{\%} and the strongest evaluated open-source VLM reaches only 31.6{\%}. To identify the source of this gap, we conduct two controlled interventions. Telling Qwen3-VL which interaction to perform raises its accuracy only from 30.8{\%} to 33.9{\%}, whereas performing the interaction for the model raises it to 81.9{\%}. These results identify reveal-interaction execution, rather than target recognition after exposure, as the main bottleneck. We further show that this capability is learnable, demonstrating that visibility recovery can be strengthened consistently across model families."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kim-etal-2026-igg">
<titleInfo>
<title>IGG: A Benchmark for Interactive GUI Grounding under Visibility Constraints</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kyeong</namePart>
<namePart type="given">Seon</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jiyeon</namePart>
<namePart type="family">Son</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tae-Hyun</namePart>
<namePart type="family">Oh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>GUI grounding benchmarks ask whether agents can localize a target that is already visible in a static screenshot. Real interfaces often violate this assumption: a target may be off-screen, appear only after a state change or hover, remain occluded, or become actionable only after a delay. This leaves the ability to recover target visibility unevaluated. To measure this reveal step, we introduce Interactive GUI Grounding (IGG), a benchmark in which an agent must first make the target visible through interaction and then click it. IGG contains 601 tasks across 16 controllable GUI environments and seven visibility-constraint sub-types, with a compact action space of scrolling, moving, dragging, waiting, and clicking. In our human study, participants achieve 94.6% accuracy, while GPT-5.5 reaches 79.4% and the strongest evaluated open-source VLM reaches only 31.6%. To identify the source of this gap, we conduct two controlled interventions. Telling Qwen3-VL which interaction to perform raises its accuracy only from 30.8% to 33.9%, whereas performing the interaction for the model raises it to 81.9%. These results identify reveal-interaction execution, rather than target recognition after exposure, as the main bottleneck. We further show that this capability is learnable, demonstrating that visibility recovery can be strengthened consistently across model families.</abstract>
<identifier type="citekey">kim-etal-2026-igg</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.2703/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>61475</start>
<end>61498</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T IGG: A Benchmark for Interactive GUI Grounding under Visibility Constraints
%A Kim, Kyeong Seon
%A Son, Jiyeon
%A Oh, Tae-Hyun
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F kim-etal-2026-igg
%X GUI grounding benchmarks ask whether agents can localize a target that is already visible in a static screenshot. Real interfaces often violate this assumption: a target may be off-screen, appear only after a state change or hover, remain occluded, or become actionable only after a delay. This leaves the ability to recover target visibility unevaluated. To measure this reveal step, we introduce Interactive GUI Grounding (IGG), a benchmark in which an agent must first make the target visible through interaction and then click it. IGG contains 601 tasks across 16 controllable GUI environments and seven visibility-constraint sub-types, with a compact action space of scrolling, moving, dragging, waiting, and clicking. In our human study, participants achieve 94.6% accuracy, while GPT-5.5 reaches 79.4% and the strongest evaluated open-source VLM reaches only 31.6%. To identify the source of this gap, we conduct two controlled interventions. Telling Qwen3-VL which interaction to perform raises its accuracy only from 30.8% to 33.9%, whereas performing the interaction for the model raises it to 81.9%. These results identify reveal-interaction execution, rather than target recognition after exposure, as the main bottleneck. We further show that this capability is learnable, demonstrating that visibility recovery can be strengthened consistently across model families.
%U https://aclanthology.org/2026.emnlp-main.2703/
%P 61475-61498
Markdown (Informal)
[IGG: A Benchmark for Interactive GUI Grounding under Visibility Constraints](https://aclanthology.org/2026.emnlp-main.2703/) (Kim et al., EMNLP 2026)
ACL