@inproceedings{wilie-etal-2026-j,
title = "{J}-{T}o{RL}: Tool-Integrated {LLM}-as-a-Judge via Reinforcement Learning",
author = "Wilie, Bryan and
YU, Tiezheng and
Dunda, Gerry Windiarto Mohamad and
Chen, Jierun and
Tao, Chaofan and
Xu, Yan and
He, Junxian and
Fung, Pascale",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.46/",
pages = "952--965",
ISBN = "979-8-89176-439-2",
abstract = "LLM-as-a-Judge frameworks offer scalable alternatives to human evaluation but inherit the limitations of text-only reasoning, particularly in tasks requiring precise computations. We introduce J-ToRL, a framework that trains LLM judges to leverage computational tools via code generation through end-to-end reinforcement learning (RL). Unlike prior approaches relying on inference-time integration, J-ToRL internalizes tool-use strategies as part of its policy, enabling exploration of diverse reasoning pathways. We identify a key challenge in applying RL with verifiable rewards to judgment: the coarse reward signals inherent in multiple-choice verdicts can incentivize shortcut learning. J-ToRL mitigates this via an auxiliary verifiable reward mechanism based on reference-answers, that enforces stricter criteria beyond simple verdict accuracy alone. With a sophisticated pairwise comparison dataset across five domains, J-ToRL-trained LLMs improve the base models, outperform prior judges, and close the gap to frontier LLMs. Further, J-ToRL-trained models exhibit beneficial cognitive behaviors such as self-error-recovery and self-refinement of tool use strategies."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wilie-etal-2026-j">
<titleInfo>
<title>J-ToRL: Tool-Integrated LLM-as-a-Judge via Reinforcement Learning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bryan</namePart>
<namePart type="family">Wilie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tiezheng</namePart>
<namePart type="family">YU</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerry</namePart>
<namePart type="given">Windiarto</namePart>
<namePart type="given">Mohamad</namePart>
<namePart type="family">Dunda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jierun</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chaofan</namePart>
<namePart type="family">Tao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Junxian</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pascale</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>LLM-as-a-Judge frameworks offer scalable alternatives to human evaluation but inherit the limitations of text-only reasoning, particularly in tasks requiring precise computations. We introduce J-ToRL, a framework that trains LLM judges to leverage computational tools via code generation through end-to-end reinforcement learning (RL). Unlike prior approaches relying on inference-time integration, J-ToRL internalizes tool-use strategies as part of its policy, enabling exploration of diverse reasoning pathways. We identify a key challenge in applying RL with verifiable rewards to judgment: the coarse reward signals inherent in multiple-choice verdicts can incentivize shortcut learning. J-ToRL mitigates this via an auxiliary verifiable reward mechanism based on reference-answers, that enforces stricter criteria beyond simple verdict accuracy alone. With a sophisticated pairwise comparison dataset across five domains, J-ToRL-trained LLMs improve the base models, outperform prior judges, and close the gap to frontier LLMs. Further, J-ToRL-trained models exhibit beneficial cognitive behaviors such as self-error-recovery and self-refinement of tool use strategies.</abstract>
<identifier type="citekey">wilie-etal-2026-j</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.46/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>952</start>
<end>965</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T J-ToRL: Tool-Integrated LLM-as-a-Judge via Reinforcement Learning
%A Wilie, Bryan
%A YU, Tiezheng
%A Dunda, Gerry Windiarto Mohamad
%A Chen, Jierun
%A Tao, Chaofan
%A Xu, Yan
%A He, Junxian
%A Fung, Pascale
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F wilie-etal-2026-j
%X LLM-as-a-Judge frameworks offer scalable alternatives to human evaluation but inherit the limitations of text-only reasoning, particularly in tasks requiring precise computations. We introduce J-ToRL, a framework that trains LLM judges to leverage computational tools via code generation through end-to-end reinforcement learning (RL). Unlike prior approaches relying on inference-time integration, J-ToRL internalizes tool-use strategies as part of its policy, enabling exploration of diverse reasoning pathways. We identify a key challenge in applying RL with verifiable rewards to judgment: the coarse reward signals inherent in multiple-choice verdicts can incentivize shortcut learning. J-ToRL mitigates this via an auxiliary verifiable reward mechanism based on reference-answers, that enforces stricter criteria beyond simple verdict accuracy alone. With a sophisticated pairwise comparison dataset across five domains, J-ToRL-trained LLMs improve the base models, outperform prior judges, and close the gap to frontier LLMs. Further, J-ToRL-trained models exhibit beneficial cognitive behaviors such as self-error-recovery and self-refinement of tool use strategies.
%U https://aclanthology.org/2026.aacl-long.46/
%P 952-965
Markdown (Informal)
[J-ToRL: Tool-Integrated LLM-as-a-Judge via Reinforcement Learning](https://aclanthology.org/2026.aacl-long.46/) (Wilie et al., AACL-IJCNLP 2026)
ACL
- Bryan Wilie, Tiezheng YU, Gerry Windiarto Mohamad Dunda, Jierun Chen, Chaofan Tao, Yan Xu, Junxian He, and Pascale Fung. 2026. J-ToRL: Tool-Integrated LLM-as-a-Judge via Reinforcement Learning. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 952–965, Hengqin, China. Association for Computational Linguistics.