@inproceedings{glass-miron-2026-comporca,
title = "{C}omp{O}rca: Corpus-Scale Compliance Labelling of Instruction-Tuning Data",
author = "Glass, Philipp E. and
Miron, Alina",
editor = "Xiong, Deyi and
Branco, Ant{\'o}nio and
Zhang, Hongming and
Dong, Yue and
Wang, Benyou and
Zhang, Wenxuan and
Zhou, Li and
Zheng, Jingting",
booktitle = "Proceedings of the First Workshop on Pluralistic Value Alignment of {LLM}s",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.plurvallm-1.16/",
pages = "211--218",
ISBN = "979-8-89176-453-8",
abstract = "Studying how fine-tuning shapes refusal and noncompliance behaviour requires knowing which training examples refuse or otherwise fail to fulfil the request. Existing annotations cover evaluation sets, which are far smaller than training corpora. We present CompOrca, compliance labels for all 4,233,923 examples of the OpenOrca corpus. Every example was classified as compliant or noncompliant by five passes of an open-weight LLM judge (LongCat-2.0, 1.6T parameters). The corpus is released as unanimous compliance (94.75{\%}), unanimous noncompliance (1.28{\%}), and nonunanimous rows (3.97{\%}), with the raw vote counts. A single pass flags 2.7{--}3.2{\%} of the corpus as noncompliant, while only 1.28{\%} is flagged by all five, so the most ambiguous rows can be filtered out. Against 450 human-annotated examples (150 annotated twice; human{--}human $\\kappa = 0.93$), the unanimous compliance and noncompliance labels are 97.3{\%} and 86.7{\%} precise. The noncompliance label is a high-precision subset of the corpus{'}s noncompliance. Published refusal-detection methods recall between 0.4{\%} and 94.1{\%} of human-labelled noncompliance. We release the full corpus with its per-row labels and vote counts at {\ensuremath{<}}https://huggingface.co/datasets/cemiu/CompOrca{\ensuremath{>}}"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="glass-miron-2026-comporca">
<titleInfo>
<title>CompOrca: Corpus-Scale Compliance Labelling of Instruction-Tuning Data</title>
</titleInfo>
<name type="personal">
<namePart type="given">Philipp</namePart>
<namePart type="given">E</namePart>
<namePart type="family">Glass</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alina</namePart>
<namePart type="family">Miron</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Pluralistic Value Alignment of LLMs</title>
</titleInfo>
<name type="personal">
<namePart type="given">Deyi</namePart>
<namePart type="family">Xiong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">António</namePart>
<namePart type="family">Branco</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hongming</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Benyou</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wenxuan</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Li</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingting</namePart>
<namePart type="family">Zheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-453-8</identifier>
</relatedItem>
<abstract>Studying how fine-tuning shapes refusal and noncompliance behaviour requires knowing which training examples refuse or otherwise fail to fulfil the request. Existing annotations cover evaluation sets, which are far smaller than training corpora. We present CompOrca, compliance labels for all 4,233,923 examples of the OpenOrca corpus. Every example was classified as compliant or noncompliant by five passes of an open-weight LLM judge (LongCat-2.0, 1.6T parameters). The corpus is released as unanimous compliance (94.75%), unanimous noncompliance (1.28%), and nonunanimous rows (3.97%), with the raw vote counts. A single pass flags 2.7–3.2% of the corpus as noncompliant, while only 1.28% is flagged by all five, so the most ambiguous rows can be filtered out. Against 450 human-annotated examples (150 annotated twice; human–human \ąppa = 0.93), the unanimous compliance and noncompliance labels are 97.3% and 86.7% precise. The noncompliance label is a high-precision subset of the corpus’s noncompliance. Published refusal-detection methods recall between 0.4% and 94.1% of human-labelled noncompliance. We release the full corpus with its per-row labels and vote counts at \ensuremath<https://huggingface.co/datasets/cemiu/CompOrca\ensuremath></abstract>
<identifier type="citekey">glass-miron-2026-comporca</identifier>
<location>
<url>https://aclanthology.org/2026.plurvallm-1.16/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>211</start>
<end>218</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T CompOrca: Corpus-Scale Compliance Labelling of Instruction-Tuning Data
%A Glass, Philipp E.
%A Miron, Alina
%Y Xiong, Deyi
%Y Branco, António
%Y Zhang, Hongming
%Y Dong, Yue
%Y Wang, Benyou
%Y Zhang, Wenxuan
%Y Zhou, Li
%Y Zheng, Jingting
%S Proceedings of the First Workshop on Pluralistic Value Alignment of LLMs
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-453-8
%F glass-miron-2026-comporca
%X Studying how fine-tuning shapes refusal and noncompliance behaviour requires knowing which training examples refuse or otherwise fail to fulfil the request. Existing annotations cover evaluation sets, which are far smaller than training corpora. We present CompOrca, compliance labels for all 4,233,923 examples of the OpenOrca corpus. Every example was classified as compliant or noncompliant by five passes of an open-weight LLM judge (LongCat-2.0, 1.6T parameters). The corpus is released as unanimous compliance (94.75%), unanimous noncompliance (1.28%), and nonunanimous rows (3.97%), with the raw vote counts. A single pass flags 2.7–3.2% of the corpus as noncompliant, while only 1.28% is flagged by all five, so the most ambiguous rows can be filtered out. Against 450 human-annotated examples (150 annotated twice; human–human \ąppa = 0.93), the unanimous compliance and noncompliance labels are 97.3% and 86.7% precise. The noncompliance label is a high-precision subset of the corpus’s noncompliance. Published refusal-detection methods recall between 0.4% and 94.1% of human-labelled noncompliance. We release the full corpus with its per-row labels and vote counts at \ensuremath<https://huggingface.co/datasets/cemiu/CompOrca\ensuremath>
%U https://aclanthology.org/2026.plurvallm-1.16/
%P 211-218
Markdown (Informal)
[CompOrca: Corpus-Scale Compliance Labelling of Instruction-Tuning Data](https://aclanthology.org/2026.plurvallm-1.16/) (Glass & Miron, PlurVA-LLM 2026)
ACL