@inproceedings{nguyen-etal-2026-scaling,
title = "Scaling {RLHF} Annotation: A Hybrid Human-{AI} Collaboration Framework",
author = "Nguyen, Khanh and
Chen, Shi and
Panguluri, Suseella and
Oliver, Jesse and
Dave, Salika and
Gundecha, Pritam and
Delalleau, Olivier",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.188/",
pages = "3161--3185",
ISBN = "979-8-89176-460-6",
abstract = "High-quality preference data collection remains a bottleneck for RLHF reward model training: collecting multiple independent human annotations per sample is costly and hard to scale across domains and languages, while fully automated AI feedback (RLAIF) risks quality degradation. We present a production-grade hybrid human-AI annotation framework that introduces automated model-as-annotator pre-annotations into a quality-control (QC)-ratified workflow, contributing model votes only when automation is sufficiently reliable and leaving final adjudication to human QC. The pipeline (1) verifies factual claims with a web-search-grounded fact checker under a cleanroom constraint that keeps proprietary tokens out of training data, (2) elicits structured judgments from an ensemble of open-weights LLM judges, and (3) applies a two-call swap-consistency gate to mitigate position bias, with a data flywheel surfacing disagreements between QC and model outputs to drive improvement. In production, the framework reduces initial-labeler hours by $\sim$37{\%}; across 11,092 production comparisons it reaches 81.6{\%} agreement with QC ($\kappa{=}0.63$) and 82{\%} to 88{\%} on the public JudgeLM benchmark, with no QC degradation measured under an independent-model proxy. The resulting data supported a production reward model and aligned LLM."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="nguyen-etal-2026-scaling">
<titleInfo>
<title>Scaling RLHF Annotation: A Hybrid Human-AI Collaboration Framework</title>
</titleInfo>
<name type="personal">
<namePart type="given">Khanh</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shi</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suseella</namePart>
<namePart type="family">Panguluri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jesse</namePart>
<namePart type="family">Oliver</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salika</namePart>
<namePart type="family">Dave</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pritam</namePart>
<namePart type="family">Gundecha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Olivier</namePart>
<namePart type="family">Delalleau</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>High-quality preference data collection remains a bottleneck for RLHF reward model training: collecting multiple independent human annotations per sample is costly and hard to scale across domains and languages, while fully automated AI feedback (RLAIF) risks quality degradation. We present a production-grade hybrid human-AI annotation framework that introduces automated model-as-annotator pre-annotations into a quality-control (QC)-ratified workflow, contributing model votes only when automation is sufficiently reliable and leaving final adjudication to human QC. The pipeline (1) verifies factual claims with a web-search-grounded fact checker under a cleanroom constraint that keeps proprietary tokens out of training data, (2) elicits structured judgments from an ensemble of open-weights LLM judges, and (3) applies a two-call swap-consistency gate to mitigate position bias, with a data flywheel surfacing disagreements between QC and model outputs to drive improvement. In production, the framework reduces initial-labeler hours by \sim37%; across 11,092 production comparisons it reaches 81.6% agreement with QC (ąppa=0.63) and 82% to 88% on the public JudgeLM benchmark, with no QC degradation measured under an independent-model proxy. The resulting data supported a production reward model and aligned LLM.</abstract>
<identifier type="citekey">nguyen-etal-2026-scaling</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.188/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>3161</start>
<end>3185</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Scaling RLHF Annotation: A Hybrid Human-AI Collaboration Framework
%A Nguyen, Khanh
%A Chen, Shi
%A Panguluri, Suseella
%A Oliver, Jesse
%A Dave, Salika
%A Gundecha, Pritam
%A Delalleau, Olivier
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F nguyen-etal-2026-scaling
%X High-quality preference data collection remains a bottleneck for RLHF reward model training: collecting multiple independent human annotations per sample is costly and hard to scale across domains and languages, while fully automated AI feedback (RLAIF) risks quality degradation. We present a production-grade hybrid human-AI annotation framework that introduces automated model-as-annotator pre-annotations into a quality-control (QC)-ratified workflow, contributing model votes only when automation is sufficiently reliable and leaving final adjudication to human QC. The pipeline (1) verifies factual claims with a web-search-grounded fact checker under a cleanroom constraint that keeps proprietary tokens out of training data, (2) elicits structured judgments from an ensemble of open-weights LLM judges, and (3) applies a two-call swap-consistency gate to mitigate position bias, with a data flywheel surfacing disagreements between QC and model outputs to drive improvement. In production, the framework reduces initial-labeler hours by \sim37%; across 11,092 production comparisons it reaches 81.6% agreement with QC (ąppa=0.63) and 82% to 88% on the public JudgeLM benchmark, with no QC degradation measured under an independent-model proxy. The resulting data supported a production reward model and aligned LLM.
%U https://aclanthology.org/2026.emnlp-industry.188/
%P 3161-3185
Markdown (Informal)
[Scaling RLHF Annotation: A Hybrid Human-AI Collaboration Framework](https://aclanthology.org/2026.emnlp-industry.188/) (Nguyen et al., EMNLP 2026)
ACL
- Khanh Nguyen, Shi Chen, Suseella Panguluri, Jesse Oliver, Salika Dave, Pritam Gundecha, and Olivier Delalleau. 2026. Scaling RLHF Annotation: A Hybrid Human-AI Collaboration Framework. In Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track, pages 3161–3185, Budapest, Hungary. Association for Computational Linguistics.