@inproceedings{marin-llobet-etal-2026-vision,
title = "Vision-Language Models Suppress Female Representations Under Ambiguous Input",
author = "Marin-Llobet, Arnau and
Henniger, Simon and
Banaji, Mahzarin R.",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1747/",
pages = "39860--39878",
ISBN = "979-8-89176-459-0",
abstract = "Alignment teaches vision-language models (VLMs) to avoid expressing demographic biases, and when gender is clearly visible they largely succeed. Far less is known about ambiguous inputs (a worker in full gear, a figure seen from behind) cases common in practice yet rarely studied. We find that minimal prompting pressure exposes occupation{--}gender defaults when prompting ambiguous input images, with models collapsing to male even for strongly female-stereotyped occupations. But do these outputs reflect what models actually encode internally? We introduce LALS (Latent Association Leaning Score), a zero-shot metric that projects visual-token activations into the model{'}s text-embedding space to measure concept associations per token and layer. Across 15 occupations, over 800 gender-ambiguous images, and four VLMs, internal representations and outputs often becomes systematically decoupled: models often encode a female association internally yet output male. Layer-wise analysis reveals an asymmetric filter (male signal amplifies end-to-end while female signal peaks mid-network and is suppressed before generation) and a color ablation shows that culturally loaded visual cues such as clothing color further modulate these internal associations."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="marin-llobet-etal-2026-vision">
<titleInfo>
<title>Vision-Language Models Suppress Female Representations Under Ambiguous Input</title>
</titleInfo>
<name type="personal">
<namePart type="given">Arnau</namePart>
<namePart type="family">Marin-Llobet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Henniger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mahzarin</namePart>
<namePart type="given">R</namePart>
<namePart type="family">Banaji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Alignment teaches vision-language models (VLMs) to avoid expressing demographic biases, and when gender is clearly visible they largely succeed. Far less is known about ambiguous inputs (a worker in full gear, a figure seen from behind) cases common in practice yet rarely studied. We find that minimal prompting pressure exposes occupation–gender defaults when prompting ambiguous input images, with models collapsing to male even for strongly female-stereotyped occupations. But do these outputs reflect what models actually encode internally? We introduce LALS (Latent Association Leaning Score), a zero-shot metric that projects visual-token activations into the model’s text-embedding space to measure concept associations per token and layer. Across 15 occupations, over 800 gender-ambiguous images, and four VLMs, internal representations and outputs often becomes systematically decoupled: models often encode a female association internally yet output male. Layer-wise analysis reveals an asymmetric filter (male signal amplifies end-to-end while female signal peaks mid-network and is suppressed before generation) and a color ablation shows that culturally loaded visual cues such as clothing color further modulate these internal associations.</abstract>
<identifier type="citekey">marin-llobet-etal-2026-vision</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1747/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>39860</start>
<end>39878</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Vision-Language Models Suppress Female Representations Under Ambiguous Input
%A Marin-Llobet, Arnau
%A Henniger, Simon
%A Banaji, Mahzarin R.
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F marin-llobet-etal-2026-vision
%X Alignment teaches vision-language models (VLMs) to avoid expressing demographic biases, and when gender is clearly visible they largely succeed. Far less is known about ambiguous inputs (a worker in full gear, a figure seen from behind) cases common in practice yet rarely studied. We find that minimal prompting pressure exposes occupation–gender defaults when prompting ambiguous input images, with models collapsing to male even for strongly female-stereotyped occupations. But do these outputs reflect what models actually encode internally? We introduce LALS (Latent Association Leaning Score), a zero-shot metric that projects visual-token activations into the model’s text-embedding space to measure concept associations per token and layer. Across 15 occupations, over 800 gender-ambiguous images, and four VLMs, internal representations and outputs often becomes systematically decoupled: models often encode a female association internally yet output male. Layer-wise analysis reveals an asymmetric filter (male signal amplifies end-to-end while female signal peaks mid-network and is suppressed before generation) and a color ablation shows that culturally loaded visual cues such as clothing color further modulate these internal associations.
%U https://aclanthology.org/2026.emnlp-main.1747/
%P 39860-39878
Markdown (Informal)
[Vision-Language Models Suppress Female Representations Under Ambiguous Input](https://aclanthology.org/2026.emnlp-main.1747/) (Marin-Llobet et al., EMNLP 2026)
ACL