@inproceedings{khan-akter-2026-evaluating,
title = "Evaluating Large Language Models on {U}rdu Idioms",
author = "Khan, Muhammad Farmal and
Akter, Mousumi",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1181/",
doi = "10.18653/2026.findings-emnlp.1181",
pages = "24956--24969",
ISBN = "979-8-89176-462-0",
abstract = "Idioms remain a persistent challenge in natural language processing due to their figurative and culturally grounded meanings, which distinguish them from literal expressions. Although recent advances in large language models (LLMs) have improved idiom handling across several languages, limited attention has been given to low-resource languages such as Urdu. In this work, we present a comprehensive benchmark for Urdu{--}English idiomatic translation, consisting of a manually verified dataset of 4,000 aligned idiom{--}sentence pairs in both Perso-Arabic (native Urdu script) and Romanized Urdu. We evaluate multiple tasks, including translation, paraphrasing, idiom span detection, and back-translation, using diverse prompting strategies such as cultural prompting, idiomatic prompting, and few-shot learning. Our findings show that frontier LLMs consistently outperform traditional neural machine translation systems across all evaluation settings, particularly in preserving figurative and metaphorical meaning. While models demonstrate relatively stable performance on native Urdu script, the absence of standardized orthography in Romanized Urdu introduces substantial challenges for consistency and idiom span detection. This work establishes a high-quality benchmark for cross-script idiomatic evaluation in Urdu and underscores the importance of prompt engineering in preserving figurative language meaning across languages."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="khan-akter-2026-evaluating">
<titleInfo>
<title>Evaluating Large Language Models on Urdu Idioms</title>
</titleInfo>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="given">Farmal</namePart>
<namePart type="family">Khan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mousumi</namePart>
<namePart type="family">Akter</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Idioms remain a persistent challenge in natural language processing due to their figurative and culturally grounded meanings, which distinguish them from literal expressions. Although recent advances in large language models (LLMs) have improved idiom handling across several languages, limited attention has been given to low-resource languages such as Urdu. In this work, we present a comprehensive benchmark for Urdu–English idiomatic translation, consisting of a manually verified dataset of 4,000 aligned idiom–sentence pairs in both Perso-Arabic (native Urdu script) and Romanized Urdu. We evaluate multiple tasks, including translation, paraphrasing, idiom span detection, and back-translation, using diverse prompting strategies such as cultural prompting, idiomatic prompting, and few-shot learning. Our findings show that frontier LLMs consistently outperform traditional neural machine translation systems across all evaluation settings, particularly in preserving figurative and metaphorical meaning. While models demonstrate relatively stable performance on native Urdu script, the absence of standardized orthography in Romanized Urdu introduces substantial challenges for consistency and idiom span detection. This work establishes a high-quality benchmark for cross-script idiomatic evaluation in Urdu and underscores the importance of prompt engineering in preserving figurative language meaning across languages.</abstract>
<identifier type="citekey">khan-akter-2026-evaluating</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1181</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1181/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>24956</start>
<end>24969</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Large Language Models on Urdu Idioms
%A Khan, Muhammad Farmal
%A Akter, Mousumi
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F khan-akter-2026-evaluating
%X Idioms remain a persistent challenge in natural language processing due to their figurative and culturally grounded meanings, which distinguish them from literal expressions. Although recent advances in large language models (LLMs) have improved idiom handling across several languages, limited attention has been given to low-resource languages such as Urdu. In this work, we present a comprehensive benchmark for Urdu–English idiomatic translation, consisting of a manually verified dataset of 4,000 aligned idiom–sentence pairs in both Perso-Arabic (native Urdu script) and Romanized Urdu. We evaluate multiple tasks, including translation, paraphrasing, idiom span detection, and back-translation, using diverse prompting strategies such as cultural prompting, idiomatic prompting, and few-shot learning. Our findings show that frontier LLMs consistently outperform traditional neural machine translation systems across all evaluation settings, particularly in preserving figurative and metaphorical meaning. While models demonstrate relatively stable performance on native Urdu script, the absence of standardized orthography in Romanized Urdu introduces substantial challenges for consistency and idiom span detection. This work establishes a high-quality benchmark for cross-script idiomatic evaluation in Urdu and underscores the importance of prompt engineering in preserving figurative language meaning across languages.
%R 10.18653/2026.findings-emnlp.1181
%U https://aclanthology.org/2026.findings-emnlp.1181/
%U https://doi.org/10.18653/2026.findings-emnlp.1181
%P 24956-24969
Markdown (Informal)
[Evaluating Large Language Models on Urdu Idioms](https://aclanthology.org/2026.findings-emnlp.1181/) (Khan & Akter, Findings 2026)
ACL
- Muhammad Farmal Khan and Mousumi Akter. 2026. Evaluating Large Language Models on Urdu Idioms. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 24956–24969, Budapest, Hungary. Association for Computational Linguistics.