@inproceedings{devane-etal-2026-bhashabench,
title = "{B}hasha{B}ench {V}1: A Comprehensive Benchmark for the Quadrant of {I}ndic Domains",
author = "Devane, Vijay and
Nauman, Mohd and
Patel, Bhargav and
Wakchoure, Aniket Mahendra and
Sant, Yogeshkumar and
Pawar, Shyam and
Thakur, Viraj and
Godse, Ananya and
Patra, Sunil and
Maurya, Neha and
Racha, Suraj and
Singh, Nitish Kamal and
Sawarkar, Piyush and
Pundalik, Kundeshwar Vijayrao and
Saluja, Rohit and
Ramakrishnan, Ganesh",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2304/",
doi = "10.18653/2026.findings-emnlp.2304",
pages = "49394--49456",
ISBN = "979-8-89176-462-0",
abstract = "The rapid advancement of large language models (LLMs) has intensified the need for domain and culture specific evaluation. Existing benchmarks are largely Anglocentric and domain-agnostic, limiting their applicability to India-centric contexts. To address this gap, we introduce BhashaBench V1, a large-scale, bilingual, exam-based benchmark covering four India-centric domains, Agriculture, Legal, Finance, and Ayurveda. BhashaBench V1 contains 74,166 meticulously curated question-answer pairs, with 52,494 in English and 21,672 in Hindi, sourced from authentic government and domain-specific exams. It is organized around a quadrant of four domains: Agriculture, Legal, Finance, and Ayurveda, comprising 90+ subdomains and covering 500+ topics, enabling fine-grained evaluation. Evaluation of 29+ LLMs reveals significant domain and language specific performance gaps, with especially large disparities in low-resource domains. For instance, GPT-4o achieves 76.49{\%} overall accuracy in Legal but only 59.74{\%} in Ayurveda. Models consistently perform better on English content compared to Hindi across all domains. Subdomain-level analysis shows that areas such as Cyber Law, International Finance perform relatively well, while Panchakarma, Seed Science, and Human Rights remain notably weak. BhashaBench V1 provides a large-scale, exam-sourced dataset for evaluating large language models across these four Indian knowledge domains. It enables assessment of models' ability to integrate domain-specific knowledge with bilingual understanding. All code, benchmarks, and resources are publicly available to support open research."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="devane-etal-2026-bhashabench">
<titleInfo>
<title>BhashaBench V1: A Comprehensive Benchmark for the Quadrant of Indic Domains</title>
</titleInfo>
<name type="personal">
<namePart type="given">Vijay</namePart>
<namePart type="family">Devane</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohd</namePart>
<namePart type="family">Nauman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bhargav</namePart>
<namePart type="family">Patel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aniket</namePart>
<namePart type="given">Mahendra</namePart>
<namePart type="family">Wakchoure</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yogeshkumar</namePart>
<namePart type="family">Sant</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shyam</namePart>
<namePart type="family">Pawar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Viraj</namePart>
<namePart type="family">Thakur</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ananya</namePart>
<namePart type="family">Godse</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunil</namePart>
<namePart type="family">Patra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neha</namePart>
<namePart type="family">Maurya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suraj</namePart>
<namePart type="family">Racha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nitish</namePart>
<namePart type="given">Kamal</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Piyush</namePart>
<namePart type="family">Sawarkar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kundeshwar</namePart>
<namePart type="given">Vijayrao</namePart>
<namePart type="family">Pundalik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rohit</namePart>
<namePart type="family">Saluja</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ganesh</namePart>
<namePart type="family">Ramakrishnan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The rapid advancement of large language models (LLMs) has intensified the need for domain and culture specific evaluation. Existing benchmarks are largely Anglocentric and domain-agnostic, limiting their applicability to India-centric contexts. To address this gap, we introduce BhashaBench V1, a large-scale, bilingual, exam-based benchmark covering four India-centric domains, Agriculture, Legal, Finance, and Ayurveda. BhashaBench V1 contains 74,166 meticulously curated question-answer pairs, with 52,494 in English and 21,672 in Hindi, sourced from authentic government and domain-specific exams. It is organized around a quadrant of four domains: Agriculture, Legal, Finance, and Ayurveda, comprising 90+ subdomains and covering 500+ topics, enabling fine-grained evaluation. Evaluation of 29+ LLMs reveals significant domain and language specific performance gaps, with especially large disparities in low-resource domains. For instance, GPT-4o achieves 76.49% overall accuracy in Legal but only 59.74% in Ayurveda. Models consistently perform better on English content compared to Hindi across all domains. Subdomain-level analysis shows that areas such as Cyber Law, International Finance perform relatively well, while Panchakarma, Seed Science, and Human Rights remain notably weak. BhashaBench V1 provides a large-scale, exam-sourced dataset for evaluating large language models across these four Indian knowledge domains. It enables assessment of models’ ability to integrate domain-specific knowledge with bilingual understanding. All code, benchmarks, and resources are publicly available to support open research.</abstract>
<identifier type="citekey">devane-etal-2026-bhashabench</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2304</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2304/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>49394</start>
<end>49456</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T BhashaBench V1: A Comprehensive Benchmark for the Quadrant of Indic Domains
%A Devane, Vijay
%A Nauman, Mohd
%A Patel, Bhargav
%A Wakchoure, Aniket Mahendra
%A Sant, Yogeshkumar
%A Pawar, Shyam
%A Thakur, Viraj
%A Godse, Ananya
%A Patra, Sunil
%A Maurya, Neha
%A Racha, Suraj
%A Singh, Nitish Kamal
%A Sawarkar, Piyush
%A Pundalik, Kundeshwar Vijayrao
%A Saluja, Rohit
%A Ramakrishnan, Ganesh
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F devane-etal-2026-bhashabench
%X The rapid advancement of large language models (LLMs) has intensified the need for domain and culture specific evaluation. Existing benchmarks are largely Anglocentric and domain-agnostic, limiting their applicability to India-centric contexts. To address this gap, we introduce BhashaBench V1, a large-scale, bilingual, exam-based benchmark covering four India-centric domains, Agriculture, Legal, Finance, and Ayurveda. BhashaBench V1 contains 74,166 meticulously curated question-answer pairs, with 52,494 in English and 21,672 in Hindi, sourced from authentic government and domain-specific exams. It is organized around a quadrant of four domains: Agriculture, Legal, Finance, and Ayurveda, comprising 90+ subdomains and covering 500+ topics, enabling fine-grained evaluation. Evaluation of 29+ LLMs reveals significant domain and language specific performance gaps, with especially large disparities in low-resource domains. For instance, GPT-4o achieves 76.49% overall accuracy in Legal but only 59.74% in Ayurveda. Models consistently perform better on English content compared to Hindi across all domains. Subdomain-level analysis shows that areas such as Cyber Law, International Finance perform relatively well, while Panchakarma, Seed Science, and Human Rights remain notably weak. BhashaBench V1 provides a large-scale, exam-sourced dataset for evaluating large language models across these four Indian knowledge domains. It enables assessment of models’ ability to integrate domain-specific knowledge with bilingual understanding. All code, benchmarks, and resources are publicly available to support open research.
%R 10.18653/2026.findings-emnlp.2304
%U https://aclanthology.org/2026.findings-emnlp.2304/
%U https://doi.org/10.18653/2026.findings-emnlp.2304
%P 49394-49456
Markdown (Informal)
[BhashaBench V1: A Comprehensive Benchmark for the Quadrant of Indic Domains](https://aclanthology.org/2026.findings-emnlp.2304/) (Devane et al., Findings 2026)
ACL
- Vijay Devane, Mohd Nauman, Bhargav Patel, Aniket Mahendra Wakchoure, Yogeshkumar Sant, Shyam Pawar, Viraj Thakur, Ananya Godse, Sunil Patra, Neha Maurya, Suraj Racha, Nitish Kamal Singh, Piyush Sawarkar, Kundeshwar Vijayrao Pundalik, Rohit Saluja, and Ganesh Ramakrishnan. 2026. BhashaBench V1: A Comprehensive Benchmark for the Quadrant of Indic Domains. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 49394–49456, Budapest, Hungary. Association for Computational Linguistics.