@inproceedings{varsha-etal-2026-evaluating,
title = "Evaluating Linguistic Knowledge of {LLM}s in {T}amil: The {ILAKKANAM} Benchmark",
author = "Varsha, Jeyarajalingam and
Velayuthan, Menan and
Karunakaran, Sumirtha and
Nivethiga, Rasan and
Sarveswaran, Kengatharaiyer",
editor = "Sarveswaran, Kengatharaiyer and
Vaidya, Ashwini",
booktitle = "Proceedings of the Second workshop on Challenges in Processing {S}outh {A}sian Languages ({CH}i{PSAL}2026)",
month = may,
year = "2026",
address = "Palma de Mallorca, Spain",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.chipsal-1.17/",
doi = "10.63317/23ob532xgibu",
pages = "176--189",
abstract = "Large Language Models (LLMs) have shown strong generalization across tasks in high-resource languages; however, their linguistic competence in low-resource and morphologically rich languages such as Tamil remains largely unexplored. Existing multilingual benchmarks often rely on translated English datasets, failing to capture the language specific linguistic and cultural nuances of the target language. To address this gap, we introduce ILAKKANAM, the first Tamil-specific linguistic evaluation benchmark manually curated using 820 questions from Sri Lankan school-level Tamil subject examination papers spanning Grades 1{--}13. Each question is annotated by trained linguists under five linguistic categories and a factual knowledge category. We evaluate both closed-source and open-source LLMs using a standardized evaluation pipeline. Our results show that Gemini 2.5 achieves the highest overall performance, while open-source models lag behind, highlighting the gap in linguistic grounding. Category- and grade-wise analyses reveal that all models perform well on lower-grade questions but show a clear decline as the grade level and the linguistic complexity of the questions increase. Further, no strong correlation is observed between a model{'}s overall performance and its ability to identify linguistic categories, suggesting that performance may be driven by exposure rather than genuine understanding. The code and dataset used in this study are publicly available in our repository, where the dataset consists only of extracted examination questions to mitigate potential data leakage. Keywords: Tamil, Linguistic Benchmark, Linguistic diagnostics, Low-resource language"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="varsha-etal-2026-evaluating">
<titleInfo>
<title>Evaluating Linguistic Knowledge of LLMs in Tamil: The ILAKKANAM Benchmark</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jeyarajalingam</namePart>
<namePart type="family">Varsha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Menan</namePart>
<namePart type="family">Velayuthan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sumirtha</namePart>
<namePart type="family">Karunakaran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rasan</namePart>
<namePart type="family">Nivethiga</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kengatharaiyer</namePart>
<namePart type="family">Sarveswaran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second workshop on Challenges in Processing South Asian Languages (CHiPSAL2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kengatharaiyer</namePart>
<namePart type="family">Sarveswaran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ashwini</namePart>
<namePart type="family">Vaidya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma de Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Large Language Models (LLMs) have shown strong generalization across tasks in high-resource languages; however, their linguistic competence in low-resource and morphologically rich languages such as Tamil remains largely unexplored. Existing multilingual benchmarks often rely on translated English datasets, failing to capture the language specific linguistic and cultural nuances of the target language. To address this gap, we introduce ILAKKANAM, the first Tamil-specific linguistic evaluation benchmark manually curated using 820 questions from Sri Lankan school-level Tamil subject examination papers spanning Grades 1–13. Each question is annotated by trained linguists under five linguistic categories and a factual knowledge category. We evaluate both closed-source and open-source LLMs using a standardized evaluation pipeline. Our results show that Gemini 2.5 achieves the highest overall performance, while open-source models lag behind, highlighting the gap in linguistic grounding. Category- and grade-wise analyses reveal that all models perform well on lower-grade questions but show a clear decline as the grade level and the linguistic complexity of the questions increase. Further, no strong correlation is observed between a model’s overall performance and its ability to identify linguistic categories, suggesting that performance may be driven by exposure rather than genuine understanding. The code and dataset used in this study are publicly available in our repository, where the dataset consists only of extracted examination questions to mitigate potential data leakage. Keywords: Tamil, Linguistic Benchmark, Linguistic diagnostics, Low-resource language</abstract>
<identifier type="citekey">varsha-etal-2026-evaluating</identifier>
<identifier type="doi">10.63317/23ob532xgibu</identifier>
<location>
<url>https://aclanthology.org/2026.chipsal-1.17/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>176</start>
<end>189</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Linguistic Knowledge of LLMs in Tamil: The ILAKKANAM Benchmark
%A Varsha, Jeyarajalingam
%A Velayuthan, Menan
%A Karunakaran, Sumirtha
%A Nivethiga, Rasan
%A Sarveswaran, Kengatharaiyer
%Y Sarveswaran, Kengatharaiyer
%Y Vaidya, Ashwini
%S Proceedings of the Second workshop on Challenges in Processing South Asian Languages (CHiPSAL2026)
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma de Mallorca, Spain
%F varsha-etal-2026-evaluating
%X Large Language Models (LLMs) have shown strong generalization across tasks in high-resource languages; however, their linguistic competence in low-resource and morphologically rich languages such as Tamil remains largely unexplored. Existing multilingual benchmarks often rely on translated English datasets, failing to capture the language specific linguistic and cultural nuances of the target language. To address this gap, we introduce ILAKKANAM, the first Tamil-specific linguistic evaluation benchmark manually curated using 820 questions from Sri Lankan school-level Tamil subject examination papers spanning Grades 1–13. Each question is annotated by trained linguists under five linguistic categories and a factual knowledge category. We evaluate both closed-source and open-source LLMs using a standardized evaluation pipeline. Our results show that Gemini 2.5 achieves the highest overall performance, while open-source models lag behind, highlighting the gap in linguistic grounding. Category- and grade-wise analyses reveal that all models perform well on lower-grade questions but show a clear decline as the grade level and the linguistic complexity of the questions increase. Further, no strong correlation is observed between a model’s overall performance and its ability to identify linguistic categories, suggesting that performance may be driven by exposure rather than genuine understanding. The code and dataset used in this study are publicly available in our repository, where the dataset consists only of extracted examination questions to mitigate potential data leakage. Keywords: Tamil, Linguistic Benchmark, Linguistic diagnostics, Low-resource language
%R 10.63317/23ob532xgibu
%U https://aclanthology.org/2026.chipsal-1.17/
%U https://doi.org/10.63317/23ob532xgibu
%P 176-189
Markdown (Informal)
[Evaluating Linguistic Knowledge of LLMs in Tamil: The ILAKKANAM Benchmark](https://aclanthology.org/2026.chipsal-1.17/) (Varsha et al., CHiPSAL 2026)
ACL
- Jeyarajalingam Varsha, Menan Velayuthan, Sumirtha Karunakaran, Rasan Nivethiga, and Kengatharaiyer Sarveswaran. 2026. Evaluating Linguistic Knowledge of LLMs in Tamil: The ILAKKANAM Benchmark. In Proceedings of the Second workshop on Challenges in Processing South Asian Languages (CHiPSAL2026), pages 176–189, Palma de Mallorca, Spain. ELRA Language Resources Association (ELRA).