@inproceedings{singh-lall-2026-probing,
title = "Probing for Negation: Extracting Compositional Negation Signals from Pretrained Language Representations for Image Retrieval",
author = "Singh, Animesh and
Lall, Brejesh",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 2: Short Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-short.35/",
pages = "525--539",
ISBN = "979-8-89176-440-8",
abstract = "Vision-language models such as CLIP exhibit strong zero-shot performance on image-text matching yet consistently fail to handle negation compositionally. Queries like ``a dog that is not running'' are collapsed into co-occurrence statistics, with the negation cue ignored. Critically, this failure is not random: because negation removes the most discriminative concept from a query, negation-blind models are systematically misled, they confidently rank the wrong images highest rather than producing an uninformative ranking. Prior work addresses this by retraining with negation-augmented data or learning embedding-level transformations. We propose vpCLIP (verypositiveCLIP), a fundamentally different approach: we extract the negation target at the query level using a frozen DistilBERT model (66M parameters). vpCLIP probes intermediate attention layers for negation-conditioned attention flow, removes candidate concept directions from hidden representations, and scores the resulting semantic shift to identify the negated concept. The extracted signal drives a retrieve-and-rerank pipeline on top of unmodified CLIP. On three curated negation benchmarks (Visual Genome, MS-COCO, Flickr30k; 800 queries total) where random retrieval averages 47-50{\%}, vpCLIP achieves 68-79{\%} Top-1 accuracy with CLIP ViT-B/32, outperforming CLIP (27-41{\%}), NegCLIP (24-37{\%}), ConCLIP (41-58{\%}), and BLIP-2 (26-34{\%}), all of which score below random on average, while requiring no task-specific training. Because the extraction stage is backbone-agnostic, vpCLIP generalizes without modification: paired with SigLIP2 SO400M it reaches 87.5{\%} on VG, 93.2{\%} on COCO, and 85.6{\%} on F30k (88.8{\%} average). We further demonstrate zero-shot transfer to an affirmation-biased medical encoder (PLIP), achieving 96{\%} Top-1 on histology images. Our results show that negation signals are latent in the intermediate representations of an auxiliary pretrained language model and require structured probing to extract, challenging the assumption that compositional negation necessitates large-scale retraining of the vision-language backbone."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="singh-lall-2026-probing">
<titleInfo>
<title>Probing for Negation: Extracting Compositional Negation Signals from Pretrained Language Representations for Image Retrieval</title>
</titleInfo>
<name type="personal">
<namePart type="given">Animesh</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Brejesh</namePart>
<namePart type="family">Lall</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-440-8</identifier>
</relatedItem>
<abstract>Vision-language models such as CLIP exhibit strong zero-shot performance on image-text matching yet consistently fail to handle negation compositionally. Queries like “a dog that is not running” are collapsed into co-occurrence statistics, with the negation cue ignored. Critically, this failure is not random: because negation removes the most discriminative concept from a query, negation-blind models are systematically misled, they confidently rank the wrong images highest rather than producing an uninformative ranking. Prior work addresses this by retraining with negation-augmented data or learning embedding-level transformations. We propose vpCLIP (verypositiveCLIP), a fundamentally different approach: we extract the negation target at the query level using a frozen DistilBERT model (66M parameters). vpCLIP probes intermediate attention layers for negation-conditioned attention flow, removes candidate concept directions from hidden representations, and scores the resulting semantic shift to identify the negated concept. The extracted signal drives a retrieve-and-rerank pipeline on top of unmodified CLIP. On three curated negation benchmarks (Visual Genome, MS-COCO, Flickr30k; 800 queries total) where random retrieval averages 47-50%, vpCLIP achieves 68-79% Top-1 accuracy with CLIP ViT-B/32, outperforming CLIP (27-41%), NegCLIP (24-37%), ConCLIP (41-58%), and BLIP-2 (26-34%), all of which score below random on average, while requiring no task-specific training. Because the extraction stage is backbone-agnostic, vpCLIP generalizes without modification: paired with SigLIP2 SO400M it reaches 87.5% on VG, 93.2% on COCO, and 85.6% on F30k (88.8% average). We further demonstrate zero-shot transfer to an affirmation-biased medical encoder (PLIP), achieving 96% Top-1 on histology images. Our results show that negation signals are latent in the intermediate representations of an auxiliary pretrained language model and require structured probing to extract, challenging the assumption that compositional negation necessitates large-scale retraining of the vision-language backbone.</abstract>
<identifier type="citekey">singh-lall-2026-probing</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-short.35/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>525</start>
<end>539</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Probing for Negation: Extracting Compositional Negation Signals from Pretrained Language Representations for Image Retrieval
%A Singh, Animesh
%A Lall, Brejesh
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-440-8
%F singh-lall-2026-probing
%X Vision-language models such as CLIP exhibit strong zero-shot performance on image-text matching yet consistently fail to handle negation compositionally. Queries like “a dog that is not running” are collapsed into co-occurrence statistics, with the negation cue ignored. Critically, this failure is not random: because negation removes the most discriminative concept from a query, negation-blind models are systematically misled, they confidently rank the wrong images highest rather than producing an uninformative ranking. Prior work addresses this by retraining with negation-augmented data or learning embedding-level transformations. We propose vpCLIP (verypositiveCLIP), a fundamentally different approach: we extract the negation target at the query level using a frozen DistilBERT model (66M parameters). vpCLIP probes intermediate attention layers for negation-conditioned attention flow, removes candidate concept directions from hidden representations, and scores the resulting semantic shift to identify the negated concept. The extracted signal drives a retrieve-and-rerank pipeline on top of unmodified CLIP. On three curated negation benchmarks (Visual Genome, MS-COCO, Flickr30k; 800 queries total) where random retrieval averages 47-50%, vpCLIP achieves 68-79% Top-1 accuracy with CLIP ViT-B/32, outperforming CLIP (27-41%), NegCLIP (24-37%), ConCLIP (41-58%), and BLIP-2 (26-34%), all of which score below random on average, while requiring no task-specific training. Because the extraction stage is backbone-agnostic, vpCLIP generalizes without modification: paired with SigLIP2 SO400M it reaches 87.5% on VG, 93.2% on COCO, and 85.6% on F30k (88.8% average). We further demonstrate zero-shot transfer to an affirmation-biased medical encoder (PLIP), achieving 96% Top-1 on histology images. Our results show that negation signals are latent in the intermediate representations of an auxiliary pretrained language model and require structured probing to extract, challenging the assumption that compositional negation necessitates large-scale retraining of the vision-language backbone.
%U https://aclanthology.org/2026.aacl-short.35/
%P 525-539
Markdown (Informal)
[Probing for Negation: Extracting Compositional Negation Signals from Pretrained Language Representations for Image Retrieval](https://aclanthology.org/2026.aacl-short.35/) (Singh & Lall, AACL-IJCNLP 2026)
ACL