@inproceedings{park-etal-2026-efficient,
title = "Efficient Iterative Retrieval with Heterogeneous Batching",
author = "Park, Dohyun and
Franke, Hubertus and
Waddington, Daniel G and
Sundararaman, Swaminathan and
Park, Yongjoo",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.28/",
pages = "612--626",
ISBN = "979-8-89176-459-0",
abstract = "Modern information retrieval increasingly employs both embedding and generative models to handle complex queries. However, current serving systems suffer from low throughput and poor GPU utilization because they execute these models in isolation. Coarse-grained partitioning, such as dedicating GPUs to specific tasks, fails to adapt to dynamic workloads and creates computational ``bubbles''. To address these, we present Orthrus, a serving system that performs heterogeneous batching within a single GPU kernel. The primary challenge lies in unifying embedding and generation workloads with conflicting computational patterns while optimizing batch composition for high performance. Orthrus addresses these challenges by employing incremental computing to match compute patterns and by adjusting batch composition in a workload-aware manner. Evaluation on four A100 GPUs shows that Orthrus achieves 1.28{\texttimes}{--}4.52{\texttimes} higher throughput and up to 52.0{\%} lower latency than various deployment scenarios based on homogeneous batching, including those built on vLLM. We release our code at [https://github.com/illinoisdata/Orthrus](https://github.com/illinoisdata/Orthrus)"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="park-etal-2026-efficient">
<titleInfo>
<title>Efficient Iterative Retrieval with Heterogeneous Batching</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dohyun</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hubertus</namePart>
<namePart type="family">Franke</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="given">G</namePart>
<namePart type="family">Waddington</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swaminathan</namePart>
<namePart type="family">Sundararaman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yongjoo</namePart>
<namePart type="family">Park</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Modern information retrieval increasingly employs both embedding and generative models to handle complex queries. However, current serving systems suffer from low throughput and poor GPU utilization because they execute these models in isolation. Coarse-grained partitioning, such as dedicating GPUs to specific tasks, fails to adapt to dynamic workloads and creates computational “bubbles”. To address these, we present Orthrus, a serving system that performs heterogeneous batching within a single GPU kernel. The primary challenge lies in unifying embedding and generation workloads with conflicting computational patterns while optimizing batch composition for high performance. Orthrus addresses these challenges by employing incremental computing to match compute patterns and by adjusting batch composition in a workload-aware manner. Evaluation on four A100 GPUs shows that Orthrus achieves 1.28×–4.52× higher throughput and up to 52.0% lower latency than various deployment scenarios based on homogeneous batching, including those built on vLLM. We release our code at [https://github.com/illinoisdata/Orthrus](https://github.com/illinoisdata/Orthrus)</abstract>
<identifier type="citekey">park-etal-2026-efficient</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.28/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>612</start>
<end>626</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Efficient Iterative Retrieval with Heterogeneous Batching
%A Park, Dohyun
%A Franke, Hubertus
%A Waddington, Daniel G.
%A Sundararaman, Swaminathan
%A Park, Yongjoo
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F park-etal-2026-efficient
%X Modern information retrieval increasingly employs both embedding and generative models to handle complex queries. However, current serving systems suffer from low throughput and poor GPU utilization because they execute these models in isolation. Coarse-grained partitioning, such as dedicating GPUs to specific tasks, fails to adapt to dynamic workloads and creates computational “bubbles”. To address these, we present Orthrus, a serving system that performs heterogeneous batching within a single GPU kernel. The primary challenge lies in unifying embedding and generation workloads with conflicting computational patterns while optimizing batch composition for high performance. Orthrus addresses these challenges by employing incremental computing to match compute patterns and by adjusting batch composition in a workload-aware manner. Evaluation on four A100 GPUs shows that Orthrus achieves 1.28×–4.52× higher throughput and up to 52.0% lower latency than various deployment scenarios based on homogeneous batching, including those built on vLLM. We release our code at [https://github.com/illinoisdata/Orthrus](https://github.com/illinoisdata/Orthrus)
%U https://aclanthology.org/2026.emnlp-main.28/
%P 612-626
Markdown (Informal)
[Efficient Iterative Retrieval with Heterogeneous Batching](https://aclanthology.org/2026.emnlp-main.28/) (Park et al., EMNLP 2026)
ACL
- Dohyun Park, Hubertus Franke, Daniel G Waddington, Swaminathan Sundararaman, and Yongjoo Park. 2026. Efficient Iterative Retrieval with Heterogeneous Batching. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pages 612–626, Budapest, Hungary. Association for Computational Linguistics.