@inproceedings{phetsri-etal-2026-thaipalibench,
title = "{T}hai{P}ali{B}ench: Benchmarking Large Language Models on {T}hai-Script {P}ali Understanding",
author = "Phetsri, Nobpharat and
Saeoueng, Anon and
Kongyoung, Sarawoot and
Viriyavit, Waranrach",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1278/",
pages = "27072--27099",
ISBN = "979-8-89176-462-0",
abstract = "The application of large language models (LLMs) to Pali written in the Thai script remains limited in accuracy due to its low-resource setting, complex grammar, and the absence of standardized evaluation benchmarks. In this paper, we introduce ThaiPaliBench, a comprehensive benchmark dataset designed to systematically evaluate LLM proficiency in Thai-script Pali. Our proposed dataset is an LLM-assisted benchmark derived from the Royal Pali Examination corpus and spans Parian Tham levels 1-9. ThaiPaliBench, comprises multiple-choice and sentence-pair tasks that evaluate grammatical knowledge, reading comprehension, natural language inference, and translation understanding. To verify the effectiveness of our benchmark, we evaluate over 10 model variants across 3 commercial and open-source LLM families, including ChatGPT, Claude, Kimi, Grok, Typhoon, GPT-OSS-120B, ThaiLLM, and SEA-LION. Our experiments show that even the best-performing model, Kimi K2.5 (1T), achieves only 68.3{\%} overall accuracy, with performance dropping substantially on grammatical reasoning and semantic inference tasks. These results highlight the unique challenges posed by Thai-script Pali and confirm that ThaiPaliBench serves as a reliable, linguistically grounded benchmark for evaluating LLM proficiency in this low-resource language. Our proposed dataset is publicly available at: https://github.com/ainrg-llm-dev/ThaiPaliBench"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="phetsri-etal-2026-thaipalibench">
<titleInfo>
<title>ThaiPaliBench: Benchmarking Large Language Models on Thai-Script Pali Understanding</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nobpharat</namePart>
<namePart type="family">Phetsri</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anon</namePart>
<namePart type="family">Saeoueng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sarawoot</namePart>
<namePart type="family">Kongyoung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Waranrach</namePart>
<namePart type="family">Viriyavit</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The application of large language models (LLMs) to Pali written in the Thai script remains limited in accuracy due to its low-resource setting, complex grammar, and the absence of standardized evaluation benchmarks. In this paper, we introduce ThaiPaliBench, a comprehensive benchmark dataset designed to systematically evaluate LLM proficiency in Thai-script Pali. Our proposed dataset is an LLM-assisted benchmark derived from the Royal Pali Examination corpus and spans Parian Tham levels 1-9. ThaiPaliBench, comprises multiple-choice and sentence-pair tasks that evaluate grammatical knowledge, reading comprehension, natural language inference, and translation understanding. To verify the effectiveness of our benchmark, we evaluate over 10 model variants across 3 commercial and open-source LLM families, including ChatGPT, Claude, Kimi, Grok, Typhoon, GPT-OSS-120B, ThaiLLM, and SEA-LION. Our experiments show that even the best-performing model, Kimi K2.5 (1T), achieves only 68.3% overall accuracy, with performance dropping substantially on grammatical reasoning and semantic inference tasks. These results highlight the unique challenges posed by Thai-script Pali and confirm that ThaiPaliBench serves as a reliable, linguistically grounded benchmark for evaluating LLM proficiency in this low-resource language. Our proposed dataset is publicly available at: https://github.com/ainrg-llm-dev/ThaiPaliBench</abstract>
<identifier type="citekey">phetsri-etal-2026-thaipalibench</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1278/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>27072</start>
<end>27099</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ThaiPaliBench: Benchmarking Large Language Models on Thai-Script Pali Understanding
%A Phetsri, Nobpharat
%A Saeoueng, Anon
%A Kongyoung, Sarawoot
%A Viriyavit, Waranrach
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F phetsri-etal-2026-thaipalibench
%X The application of large language models (LLMs) to Pali written in the Thai script remains limited in accuracy due to its low-resource setting, complex grammar, and the absence of standardized evaluation benchmarks. In this paper, we introduce ThaiPaliBench, a comprehensive benchmark dataset designed to systematically evaluate LLM proficiency in Thai-script Pali. Our proposed dataset is an LLM-assisted benchmark derived from the Royal Pali Examination corpus and spans Parian Tham levels 1-9. ThaiPaliBench, comprises multiple-choice and sentence-pair tasks that evaluate grammatical knowledge, reading comprehension, natural language inference, and translation understanding. To verify the effectiveness of our benchmark, we evaluate over 10 model variants across 3 commercial and open-source LLM families, including ChatGPT, Claude, Kimi, Grok, Typhoon, GPT-OSS-120B, ThaiLLM, and SEA-LION. Our experiments show that even the best-performing model, Kimi K2.5 (1T), achieves only 68.3% overall accuracy, with performance dropping substantially on grammatical reasoning and semantic inference tasks. These results highlight the unique challenges posed by Thai-script Pali and confirm that ThaiPaliBench serves as a reliable, linguistically grounded benchmark for evaluating LLM proficiency in this low-resource language. Our proposed dataset is publicly available at: https://github.com/ainrg-llm-dev/ThaiPaliBench
%U https://aclanthology.org/2026.findings-emnlp.1278/
%P 27072-27099
Markdown (Informal)
[ThaiPaliBench: Benchmarking Large Language Models on Thai-Script Pali Understanding](https://aclanthology.org/2026.findings-emnlp.1278/) (Phetsri et al., Findings 2026)
ACL