@inproceedings{shafique-etal-2026-urdubench,
title = "{U}rdu{B}ench: An {U}rdu Reasoning Benchmark using Contextually Ensembled Translations with Human-in-the-Loop",
author = "Shafique, Muhammad Ali and
Mehboob, Areej and
Fiaz, Layba and
Qadeer, Muhammad Usman and
Farooq, Hamza",
editor = "Ranaldi, Leonardo and
Pucci, Giulia and
Ranaldi, Federico and
Valentino, Marco and
Birch, Alexandra and
Zanzotto, Fabio Massimo",
booktitle = "The Proceedings of the Workshop on Open Reasoning Across Cultures Languages ({ORACLE})",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.oracle-main.19/",
pages = "286--303",
ISBN = "979-8-89176-483-5",
abstract = "Recent advances in large language models (LLMs) have led to strong reasoning capabilities; however, evaluating such models in lowresource languages remains challenging due to the lack of standardized reasoning benchmarks. Urdu reasoning evaluation, in particular, is limited by machine translation sensitivity and an emphasis on general language tasks rather than reasoning benchmarks. We propose a contextually ensembled translation framework with human-in-the-loop validation that leverages multiple translation systems to develop Urdu reasoning benchmarks while preserving contextual and structural integrity. Using this framework, we translate widely adopted benchmarks, including MGSM, MATH-500, CommonSenseQA (CSQA), and OpenBookQA (OBQA), into Urdu, collectively referred to as UrduBench, and evaluate reasoning-oriented and instruction-tuned LLMs across multiple prompting strategies. Our analysis examines four datasets, five task difficulty levels, diverse model architectures, model scaling settings, and language consistency. We find that multistep and symbolic reasoning tasks pose significant challenges in Urdu, and that stable language alignment is a critical prerequisite for robust reasoning. Overall, our work establishes a scalable methodology for standardized Urdu reasoning evaluation and provides insights into multilingual reasoning failures, with applicability to other low-resource languages. The code and datasets will be publicly released."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shafique-etal-2026-urdubench">
<titleInfo>
<title>UrduBench: An Urdu Reasoning Benchmark using Contextually Ensembled Translations with Human-in-the-Loop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="given">Ali</namePart>
<namePart type="family">Shafique</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Areej</namePart>
<namePart type="family">Mehboob</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Layba</namePart>
<namePart type="family">Fiaz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="given">Usman</namePart>
<namePart type="family">Qadeer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hamza</namePart>
<namePart type="family">Farooq</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>The Proceedings of the Workshop on Open Reasoning Across Cultures Languages (ORACLE)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leonardo</namePart>
<namePart type="family">Ranaldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giulia</namePart>
<namePart type="family">Pucci</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Federico</namePart>
<namePart type="family">Ranaldi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marco</namePart>
<namePart type="family">Valentino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alexandra</namePart>
<namePart type="family">Birch</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabio</namePart>
<namePart type="given">Massimo</namePart>
<namePart type="family">Zanzotto</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-483-5</identifier>
</relatedItem>
<abstract>Recent advances in large language models (LLMs) have led to strong reasoning capabilities; however, evaluating such models in lowresource languages remains challenging due to the lack of standardized reasoning benchmarks. Urdu reasoning evaluation, in particular, is limited by machine translation sensitivity and an emphasis on general language tasks rather than reasoning benchmarks. We propose a contextually ensembled translation framework with human-in-the-loop validation that leverages multiple translation systems to develop Urdu reasoning benchmarks while preserving contextual and structural integrity. Using this framework, we translate widely adopted benchmarks, including MGSM, MATH-500, CommonSenseQA (CSQA), and OpenBookQA (OBQA), into Urdu, collectively referred to as UrduBench, and evaluate reasoning-oriented and instruction-tuned LLMs across multiple prompting strategies. Our analysis examines four datasets, five task difficulty levels, diverse model architectures, model scaling settings, and language consistency. We find that multistep and symbolic reasoning tasks pose significant challenges in Urdu, and that stable language alignment is a critical prerequisite for robust reasoning. Overall, our work establishes a scalable methodology for standardized Urdu reasoning evaluation and provides insights into multilingual reasoning failures, with applicability to other low-resource languages. The code and datasets will be publicly released.</abstract>
<identifier type="citekey">shafique-etal-2026-urdubench</identifier>
<location>
<url>https://aclanthology.org/2026.oracle-main.19/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>286</start>
<end>303</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T UrduBench: An Urdu Reasoning Benchmark using Contextually Ensembled Translations with Human-in-the-Loop
%A Shafique, Muhammad Ali
%A Mehboob, Areej
%A Fiaz, Layba
%A Qadeer, Muhammad Usman
%A Farooq, Hamza
%Y Ranaldi, Leonardo
%Y Pucci, Giulia
%Y Ranaldi, Federico
%Y Valentino, Marco
%Y Birch, Alexandra
%Y Zanzotto, Fabio Massimo
%S The Proceedings of the Workshop on Open Reasoning Across Cultures Languages (ORACLE)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-483-5
%F shafique-etal-2026-urdubench
%X Recent advances in large language models (LLMs) have led to strong reasoning capabilities; however, evaluating such models in lowresource languages remains challenging due to the lack of standardized reasoning benchmarks. Urdu reasoning evaluation, in particular, is limited by machine translation sensitivity and an emphasis on general language tasks rather than reasoning benchmarks. We propose a contextually ensembled translation framework with human-in-the-loop validation that leverages multiple translation systems to develop Urdu reasoning benchmarks while preserving contextual and structural integrity. Using this framework, we translate widely adopted benchmarks, including MGSM, MATH-500, CommonSenseQA (CSQA), and OpenBookQA (OBQA), into Urdu, collectively referred to as UrduBench, and evaluate reasoning-oriented and instruction-tuned LLMs across multiple prompting strategies. Our analysis examines four datasets, five task difficulty levels, diverse model architectures, model scaling settings, and language consistency. We find that multistep and symbolic reasoning tasks pose significant challenges in Urdu, and that stable language alignment is a critical prerequisite for robust reasoning. Overall, our work establishes a scalable methodology for standardized Urdu reasoning evaluation and provides insights into multilingual reasoning failures, with applicability to other low-resource languages. The code and datasets will be publicly released.
%U https://aclanthology.org/2026.oracle-main.19/
%P 286-303
Markdown (Informal)
[UrduBench: An Urdu Reasoning Benchmark using Contextually Ensembled Translations with Human-in-the-Loop](https://aclanthology.org/2026.oracle-main.19/) (Shafique et al., ORACLE 2026)
ACL