@inproceedings{hossain-etal-2026-structural,
title = "Structural vs Cultural Reasoning: Reliability Limits of {LLM}-as-a-Judge in Multilingual Evaluation",
author = "Hossain, Shehenaz and
Bouamor, Houda and
Afli, Haithem",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.249/",
pages = "5555--5588",
ISBN = "979-8-89176-439-2",
abstract = "LLM-as-a-Judge has emerged as a scalable alternative to human evaluation in multilingual cultural reasoning, yet its reliability for culturally grounded judgments remains poorly understood. We present a human-aligned validation study using the CRaFT framework across Arabic, Bengali, and Spanish, evaluating six judge models, including five open-weight models and GPT-4.1, against paired human annotations across two structurally observable dimensions (Answer Consistency and Explanation Consistency) and three culturally interpretive dimensions (Cultural Fluency, Linguistic Adaptation, and Deviation). Our results reveal a consistent reliability divide: automated judges align substantially better with humans on structural dimensions than on culturally interpretive ones requiring sociocultural reasoning. Reliability further varies across languages, judge architectures, and response-generation models, with several failures reflecting systematic calibration limits rather than prompt underspecification alone. These findings suggest that current automated evaluators are best treated as complementary diagnostic tools rather than replacements for human judgment in culturally sensitive multilingual settings."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hossain-etal-2026-structural">
<titleInfo>
<title>Structural vs Cultural Reasoning: Reliability Limits of LLM-as-a-Judge in Multilingual Evaluation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Shehenaz</namePart>
<namePart type="family">Hossain</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haithem</namePart>
<namePart type="family">Afli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>LLM-as-a-Judge has emerged as a scalable alternative to human evaluation in multilingual cultural reasoning, yet its reliability for culturally grounded judgments remains poorly understood. We present a human-aligned validation study using the CRaFT framework across Arabic, Bengali, and Spanish, evaluating six judge models, including five open-weight models and GPT-4.1, against paired human annotations across two structurally observable dimensions (Answer Consistency and Explanation Consistency) and three culturally interpretive dimensions (Cultural Fluency, Linguistic Adaptation, and Deviation). Our results reveal a consistent reliability divide: automated judges align substantially better with humans on structural dimensions than on culturally interpretive ones requiring sociocultural reasoning. Reliability further varies across languages, judge architectures, and response-generation models, with several failures reflecting systematic calibration limits rather than prompt underspecification alone. These findings suggest that current automated evaluators are best treated as complementary diagnostic tools rather than replacements for human judgment in culturally sensitive multilingual settings.</abstract>
<identifier type="citekey">hossain-etal-2026-structural</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.249/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>5555</start>
<end>5588</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Structural vs Cultural Reasoning: Reliability Limits of LLM-as-a-Judge in Multilingual Evaluation
%A Hossain, Shehenaz
%A Bouamor, Houda
%A Afli, Haithem
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F hossain-etal-2026-structural
%X LLM-as-a-Judge has emerged as a scalable alternative to human evaluation in multilingual cultural reasoning, yet its reliability for culturally grounded judgments remains poorly understood. We present a human-aligned validation study using the CRaFT framework across Arabic, Bengali, and Spanish, evaluating six judge models, including five open-weight models and GPT-4.1, against paired human annotations across two structurally observable dimensions (Answer Consistency and Explanation Consistency) and three culturally interpretive dimensions (Cultural Fluency, Linguistic Adaptation, and Deviation). Our results reveal a consistent reliability divide: automated judges align substantially better with humans on structural dimensions than on culturally interpretive ones requiring sociocultural reasoning. Reliability further varies across languages, judge architectures, and response-generation models, with several failures reflecting systematic calibration limits rather than prompt underspecification alone. These findings suggest that current automated evaluators are best treated as complementary diagnostic tools rather than replacements for human judgment in culturally sensitive multilingual settings.
%U https://aclanthology.org/2026.aacl-long.249/
%P 5555-5588
Markdown (Informal)
[Structural vs Cultural Reasoning: Reliability Limits of LLM-as-a-Judge in Multilingual Evaluation](https://aclanthology.org/2026.aacl-long.249/) (Hossain et al., AACL-IJCNLP 2026)
ACL