@inproceedings{isaeva-etal-2026-pollux,
title = "Pollux: A Compact Judge and Workflow for Criteria-Based {LLM} Evaluation",
author = "Isaeva, Ulyana and
Egor, Maslennikov and
Glushko, Alisa and
Martynov, Nikita and
Mordasheva, Anastasia and
Dmitriy, Gorbetskiy and
Basyrova, Elina and
Skachkov, Sergey and
Victoria, Berestova and
Fenogenova, Alena and
Astafurov, Danil",
editor = "Zhang, Ningyu and
Wijaya, Derry",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics: System Demonstrations",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-demo.7/",
pages = "50--57",
ISBN = "979-8-89176-441-5",
abstract = "Evaluating open-ended generation remains non-trivial because quality varies across style, correctness, grounding, and safety. Moreover, a fixed prompt-response abstraction is insufficient for a reusable evaluation system as deployed applications increasingly expose multi-turn dialogues, retrieved evidence, and tool-calling traces. We present Pollux, a universal criteria-based evaluation framework centered on Pollux Judge, a compact Qwen3-4B-based LLM-as-a-Judge that evaluates such heterogeneous interactions through free-form user-defined criteria, scales, and output formats. Pollux follows the criteria methodology represented in the Pollux Benchmark (35 task groups, 2,115 expert-authored prompts), using it as a human-grounded resource for judge validation. The judge is trained with GRPO on Russian- and English-language data and is designed for bilingual deployment through a language-independent interaction schema. On the held-out expert-labeled test set, Pollux reaches a Spearman correlation of 0.744, exceeding every evaluated general-purpose baseline, including models with up to 1.1T parameters, while remaining suitable for local or OpenAI-compatible serving.\par System demonstration video: https://drive.google.com/file/d/1PF6L-RcrS24JVUdaVRi01j237yTdftEd/"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="isaeva-etal-2026-pollux">
<titleInfo>
<title>Pollux: A Compact Judge and Workflow for Criteria-Based LLM Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ulyana</namePart>
<namePart type="family">Isaeva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maslennikov</namePart>
<namePart type="family">Egor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alisa</namePart>
<namePart type="family">Glushko</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikita</namePart>
<namePart type="family">Martynov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anastasia</namePart>
<namePart type="family">Mordasheva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gorbetskiy</namePart>
<namePart type="family">Dmitriy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elina</namePart>
<namePart type="family">Basyrova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sergey</namePart>
<namePart type="family">Skachkov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Berestova</namePart>
<namePart type="family">Victoria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alena</namePart>
<namePart type="family">Fenogenova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Danil</namePart>
<namePart type="family">Astafurov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: System Demonstrations</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ningyu</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Derry</namePart>
<namePart type="family">Wijaya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-441-5</identifier>
</relatedItem>
<abstract>Evaluating open-ended generation remains non-trivial because quality varies across style, correctness, grounding, and safety. Moreover, a fixed prompt-response abstraction is insufficient for a reusable evaluation system as deployed applications increasingly expose multi-turn dialogues, retrieved evidence, and tool-calling traces. We present Pollux, a universal criteria-based evaluation framework centered on Pollux Judge, a compact Qwen3-4B-based LLM-as-a-Judge that evaluates such heterogeneous interactions through free-form user-defined criteria, scales, and output formats. Pollux follows the criteria methodology represented in the Pollux Benchmark (35 task groups, 2,115 expert-authored prompts), using it as a human-grounded resource for judge validation. The judge is trained with GRPO on Russian- and English-language data and is designed for bilingual deployment through a language-independent interaction schema. On the held-out expert-labeled test set, Pollux reaches a Spearman correlation of 0.744, exceeding every evaluated general-purpose baseline, including models with up to 1.1T parameters, while remaining suitable for local or OpenAI-compatible serving.\par System demonstration video: https://drive.google.com/file/d/1PF6L-RcrS24JVUdaVRi01j237yTdftEd/</abstract>
<identifier type="citekey">isaeva-etal-2026-pollux</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-demo.7/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>50</start>
<end>57</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Pollux: A Compact Judge and Workflow for Criteria-Based LLM Evaluation
%A Isaeva, Ulyana
%A Egor, Maslennikov
%A Glushko, Alisa
%A Martynov, Nikita
%A Mordasheva, Anastasia
%A Dmitriy, Gorbetskiy
%A Basyrova, Elina
%A Skachkov, Sergey
%A Victoria, Berestova
%A Fenogenova, Alena
%A Astafurov, Danil
%Y Zhang, Ningyu
%Y Wijaya, Derry
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: System Demonstrations
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-441-5
%F isaeva-etal-2026-pollux
%X Evaluating open-ended generation remains non-trivial because quality varies across style, correctness, grounding, and safety. Moreover, a fixed prompt-response abstraction is insufficient for a reusable evaluation system as deployed applications increasingly expose multi-turn dialogues, retrieved evidence, and tool-calling traces. We present Pollux, a universal criteria-based evaluation framework centered on Pollux Judge, a compact Qwen3-4B-based LLM-as-a-Judge that evaluates such heterogeneous interactions through free-form user-defined criteria, scales, and output formats. Pollux follows the criteria methodology represented in the Pollux Benchmark (35 task groups, 2,115 expert-authored prompts), using it as a human-grounded resource for judge validation. The judge is trained with GRPO on Russian- and English-language data and is designed for bilingual deployment through a language-independent interaction schema. On the held-out expert-labeled test set, Pollux reaches a Spearman correlation of 0.744, exceeding every evaluated general-purpose baseline, including models with up to 1.1T parameters, while remaining suitable for local or OpenAI-compatible serving.\par System demonstration video: https://drive.google.com/file/d/1PF6L-RcrS24JVUdaVRi01j237yTdftEd/
%U https://aclanthology.org/2026.aacl-demo.7/
%P 50-57
Markdown (Informal)
[Pollux: A Compact Judge and Workflow for Criteria-Based LLM Evaluation](https://aclanthology.org/2026.aacl-demo.7/) (Isaeva et al., AACL-IJCNLP 2026)
ACL
- Ulyana Isaeva, Maslennikov Egor, Alisa Glushko, Nikita Martynov, Anastasia Mordasheva, Gorbetskiy Dmitriy, Elina Basyrova, Sergey Skachkov, Berestova Victoria, Alena Fenogenova, and Danil Astafurov. 2026. Pollux: A Compact Judge and Workflow for Criteria-Based LLM Evaluation. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: System Demonstrations, pages 50–57, Hengqin, China. Association for Computational Linguistics.