@inproceedings{farid-etal-2026-cpjudgebench,
title = "{CPJ}udge{B}ench: Meta-Evaluation of {LLM} Judges for Constraint Programming through Solution-Space Semantics",
author = "Farid, Nastaran Moradzadeh and
Taghizadeh, Alireza and
Di Noia, Tommaso and
Deldjoo, Yashar and
Shafiee, Sara",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1648/",
pages = "37459--37478",
ISBN = "979-8-89176-459-0",
abstract = "Large Language Models (LLMs) are increasingly used as evaluators for code generation, but existing judge benchmarks mainly target general-purpose programming. Constraint Programming (CP) requires a stricter notion of correctness: a model is correct only if it faithfully represents the intended solution space, not merely if it executes, preserves satisfiability, or returns one valid solution. We introduce CPJudgeBench, a benchmark for meta-evaluating LLM judges of CP modeling correctness through solver-supported solution-space semantics. It contains 522 solver-validated candidate models from 82 CP problem families in CPMpy and MiniZinc, labeled as equivalent, unsound, incomplete, mixed, or non-executable. Across our experiments, we meta-evaluate a total of 15 LLM judges through semantic-label, binary-correctness, and score-based protocols. Results show that fine-grained semantic judging remains difficult, especially for reference-free MiniZinc, while stronger reasoning- and code-oriented models perform more reliably. Score-based LLM judgments also align better with solution-space similarity than standard text or code metrics. The benchmark system is available at https://github.com/Nastaran95/CPJudgeBench."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="farid-etal-2026-cpjudgebench">
<titleInfo>
<title>CPJudgeBench: Meta-Evaluation of LLM Judges for Constraint Programming through Solution-Space Semantics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nastaran</namePart>
<namePart type="given">Moradzadeh</namePart>
<namePart type="family">Farid</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alireza</namePart>
<namePart type="family">Taghizadeh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tommaso</namePart>
<namePart type="family">Di Noia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yashar</namePart>
<namePart type="family">Deldjoo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sara</namePart>
<namePart type="family">Shafiee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Large Language Models (LLMs) are increasingly used as evaluators for code generation, but existing judge benchmarks mainly target general-purpose programming. Constraint Programming (CP) requires a stricter notion of correctness: a model is correct only if it faithfully represents the intended solution space, not merely if it executes, preserves satisfiability, or returns one valid solution. We introduce CPJudgeBench, a benchmark for meta-evaluating LLM judges of CP modeling correctness through solver-supported solution-space semantics. It contains 522 solver-validated candidate models from 82 CP problem families in CPMpy and MiniZinc, labeled as equivalent, unsound, incomplete, mixed, or non-executable. Across our experiments, we meta-evaluate a total of 15 LLM judges through semantic-label, binary-correctness, and score-based protocols. Results show that fine-grained semantic judging remains difficult, especially for reference-free MiniZinc, while stronger reasoning- and code-oriented models perform more reliably. Score-based LLM judgments also align better with solution-space similarity than standard text or code metrics. The benchmark system is available at https://github.com/Nastaran95/CPJudgeBench.</abstract>
<identifier type="citekey">farid-etal-2026-cpjudgebench</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1648/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>37459</start>
<end>37478</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T CPJudgeBench: Meta-Evaluation of LLM Judges for Constraint Programming through Solution-Space Semantics
%A Farid, Nastaran Moradzadeh
%A Taghizadeh, Alireza
%A Di Noia, Tommaso
%A Deldjoo, Yashar
%A Shafiee, Sara
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F farid-etal-2026-cpjudgebench
%X Large Language Models (LLMs) are increasingly used as evaluators for code generation, but existing judge benchmarks mainly target general-purpose programming. Constraint Programming (CP) requires a stricter notion of correctness: a model is correct only if it faithfully represents the intended solution space, not merely if it executes, preserves satisfiability, or returns one valid solution. We introduce CPJudgeBench, a benchmark for meta-evaluating LLM judges of CP modeling correctness through solver-supported solution-space semantics. It contains 522 solver-validated candidate models from 82 CP problem families in CPMpy and MiniZinc, labeled as equivalent, unsound, incomplete, mixed, or non-executable. Across our experiments, we meta-evaluate a total of 15 LLM judges through semantic-label, binary-correctness, and score-based protocols. Results show that fine-grained semantic judging remains difficult, especially for reference-free MiniZinc, while stronger reasoning- and code-oriented models perform more reliably. Score-based LLM judgments also align better with solution-space similarity than standard text or code metrics. The benchmark system is available at https://github.com/Nastaran95/CPJudgeBench.
%U https://aclanthology.org/2026.emnlp-main.1648/
%P 37459-37478
Markdown (Informal)
[CPJudgeBench: Meta-Evaluation of LLM Judges for Constraint Programming through Solution-Space Semantics](https://aclanthology.org/2026.emnlp-main.1648/) (Farid et al., EMNLP 2026)
ACL