@inproceedings{sinhababu-etal-2026-modeling,
title = "Modeling Ranking Properties with In-Context Learning",
author = "Sinhababu, Nilanjan and
Parry, Andrew and
Ganguly, Debasis and
Mitra, Pabitra",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2297/",
doi = "10.18653/2026.findings-emnlp.2297",
pages = "49256--49271",
ISBN = "979-8-89176-462-0",
abstract = "While standard IR models are primarily designed to optimize relevance, real-world search often needs to balance additional objectives such as diversity and fairness. These objectives depend on inter-document interactions and are commonly addressed using post-hoc heuristics or supervised learning methods, which require task-specific training for each ranking scenario and dataset. In this work, we propose an in-context learning (ICL) approach for listwise LLM rerankers that eliminates the need for such training. Instead, our method relies on a small number of example rankings that demonstrate the desired trade-offs between objectives for past queries similar to the current input. We evaluate our approach on common IR test collections to investigate multiple auxiliary objectives: group fairness (TREC Fairness), polarity diversity (Touch{\'e}), and topical diversity (TREC Deep Learning 2019/2020). We empirically validate that our method enables control over ranking behavior through demonstration engineering, allowing nuanced behavioral adjustments without explicit optimization. Our experiment demonstrates significant improvements in auxiliary objectives, with up to 23{\%} in topical diversity and 20.6{\%} fairness gains across different tasks, while maintaining relevance across benchmarks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="sinhababu-etal-2026-modeling">
<titleInfo>
<title>Modeling Ranking Properties with In-Context Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nilanjan</namePart>
<namePart type="family">Sinhababu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andrew</namePart>
<namePart type="family">Parry</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Debasis</namePart>
<namePart type="family">Ganguly</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pabitra</namePart>
<namePart type="family">Mitra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>While standard IR models are primarily designed to optimize relevance, real-world search often needs to balance additional objectives such as diversity and fairness. These objectives depend on inter-document interactions and are commonly addressed using post-hoc heuristics or supervised learning methods, which require task-specific training for each ranking scenario and dataset. In this work, we propose an in-context learning (ICL) approach for listwise LLM rerankers that eliminates the need for such training. Instead, our method relies on a small number of example rankings that demonstrate the desired trade-offs between objectives for past queries similar to the current input. We evaluate our approach on common IR test collections to investigate multiple auxiliary objectives: group fairness (TREC Fairness), polarity diversity (Touché), and topical diversity (TREC Deep Learning 2019/2020). We empirically validate that our method enables control over ranking behavior through demonstration engineering, allowing nuanced behavioral adjustments without explicit optimization. Our experiment demonstrates significant improvements in auxiliary objectives, with up to 23% in topical diversity and 20.6% fairness gains across different tasks, while maintaining relevance across benchmarks.</abstract>
<identifier type="citekey">sinhababu-etal-2026-modeling</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2297</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2297/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>49256</start>
<end>49271</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Modeling Ranking Properties with In-Context Learning
%A Sinhababu, Nilanjan
%A Parry, Andrew
%A Ganguly, Debasis
%A Mitra, Pabitra
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F sinhababu-etal-2026-modeling
%X While standard IR models are primarily designed to optimize relevance, real-world search often needs to balance additional objectives such as diversity and fairness. These objectives depend on inter-document interactions and are commonly addressed using post-hoc heuristics or supervised learning methods, which require task-specific training for each ranking scenario and dataset. In this work, we propose an in-context learning (ICL) approach for listwise LLM rerankers that eliminates the need for such training. Instead, our method relies on a small number of example rankings that demonstrate the desired trade-offs between objectives for past queries similar to the current input. We evaluate our approach on common IR test collections to investigate multiple auxiliary objectives: group fairness (TREC Fairness), polarity diversity (Touché), and topical diversity (TREC Deep Learning 2019/2020). We empirically validate that our method enables control over ranking behavior through demonstration engineering, allowing nuanced behavioral adjustments without explicit optimization. Our experiment demonstrates significant improvements in auxiliary objectives, with up to 23% in topical diversity and 20.6% fairness gains across different tasks, while maintaining relevance across benchmarks.
%R 10.18653/2026.findings-emnlp.2297
%U https://aclanthology.org/2026.findings-emnlp.2297/
%U https://doi.org/10.18653/2026.findings-emnlp.2297
%P 49256-49271
Markdown (Informal)
[Modeling Ranking Properties with In-Context Learning](https://aclanthology.org/2026.findings-emnlp.2297/) (Sinhababu et al., Findings 2026)
ACL
- Nilanjan Sinhababu, Andrew Parry, Debasis Ganguly, and Pabitra Mitra. 2026. Modeling Ranking Properties with In-Context Learning. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 49256–49271, Budapest, Hungary. Association for Computational Linguistics.