@inproceedings{maji-bhattacharya-2026-arjun,
title = "{ARJUN}: A Parameter-Efficient Multi-Objective Pre-Training Framework for Language Models",
author = "Maji, Subhadip and
Bhattacharya, Arnab",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1877/",
doi = "10.18653/2026.findings-emnlp.1877",
pages = "40035--40052",
ISBN = "979-8-89176-462-0",
abstract = "The proliferation of large-scale language models has significantly advanced natural language processing capabilities; yet their substantial computational requirements create barriers for deployment in resource-constrained environments and limit accessibility for under-represented language communities. To address this fundamental issue, we present a novel multi-objective pre-training approach for developing parameter-efficient language models. Our method, implemented in the ARJUN model family, strategically combines four complementary pre-training objectives within a unified transformer encoder-decoder framework: (1) Masked Token Prediction (MT) for bidirectional understanding, (2) Next Token Prediction (NT) for autoregressive generation, (3) Masked Span Prediction (MS) for multi-token dependencies, and (4) Complete Sentence Prediction (CS) to bridge discriminative and generative tasks. Using this approach, our ARJUN-Base model (69M parameters) consistently outperforms standard baselines 2.5-4x larger. Extending this architecture, our scaled-up ARJUN-Large variant (up to 256M parameters) demonstrates superior architectural efficiency, establishing strong performance baselines within matched parameter budgets. Evaluations across 20 discriminative and generative tasks in Bangla, Tamil, and Hindi show prominent gains, including up to +6.5 percentage points (pp) over comparable baselines on macro-F1 scores and +17.5 pp sacreBLEU on generation. Finally, ablation studies confirm that joint training on these four objectives yields synergistic representations that consistently outperform single-task configurations across all scales."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="maji-bhattacharya-2026-arjun">
<titleInfo>
<title>ARJUN: A Parameter-Efficient Multi-Objective Pre-Training Framework for Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Subhadip</namePart>
<namePart type="family">Maji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Arnab</namePart>
<namePart type="family">Bhattacharya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The proliferation of large-scale language models has significantly advanced natural language processing capabilities; yet their substantial computational requirements create barriers for deployment in resource-constrained environments and limit accessibility for under-represented language communities. To address this fundamental issue, we present a novel multi-objective pre-training approach for developing parameter-efficient language models. Our method, implemented in the ARJUN model family, strategically combines four complementary pre-training objectives within a unified transformer encoder-decoder framework: (1) Masked Token Prediction (MT) for bidirectional understanding, (2) Next Token Prediction (NT) for autoregressive generation, (3) Masked Span Prediction (MS) for multi-token dependencies, and (4) Complete Sentence Prediction (CS) to bridge discriminative and generative tasks. Using this approach, our ARJUN-Base model (69M parameters) consistently outperforms standard baselines 2.5-4x larger. Extending this architecture, our scaled-up ARJUN-Large variant (up to 256M parameters) demonstrates superior architectural efficiency, establishing strong performance baselines within matched parameter budgets. Evaluations across 20 discriminative and generative tasks in Bangla, Tamil, and Hindi show prominent gains, including up to +6.5 percentage points (pp) over comparable baselines on macro-F1 scores and +17.5 pp sacreBLEU on generation. Finally, ablation studies confirm that joint training on these four objectives yields synergistic representations that consistently outperform single-task configurations across all scales.</abstract>
<identifier type="citekey">maji-bhattacharya-2026-arjun</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.1877</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1877/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>40035</start>
<end>40052</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ARJUN: A Parameter-Efficient Multi-Objective Pre-Training Framework for Language Models
%A Maji, Subhadip
%A Bhattacharya, Arnab
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F maji-bhattacharya-2026-arjun
%X The proliferation of large-scale language models has significantly advanced natural language processing capabilities; yet their substantial computational requirements create barriers for deployment in resource-constrained environments and limit accessibility for under-represented language communities. To address this fundamental issue, we present a novel multi-objective pre-training approach for developing parameter-efficient language models. Our method, implemented in the ARJUN model family, strategically combines four complementary pre-training objectives within a unified transformer encoder-decoder framework: (1) Masked Token Prediction (MT) for bidirectional understanding, (2) Next Token Prediction (NT) for autoregressive generation, (3) Masked Span Prediction (MS) for multi-token dependencies, and (4) Complete Sentence Prediction (CS) to bridge discriminative and generative tasks. Using this approach, our ARJUN-Base model (69M parameters) consistently outperforms standard baselines 2.5-4x larger. Extending this architecture, our scaled-up ARJUN-Large variant (up to 256M parameters) demonstrates superior architectural efficiency, establishing strong performance baselines within matched parameter budgets. Evaluations across 20 discriminative and generative tasks in Bangla, Tamil, and Hindi show prominent gains, including up to +6.5 percentage points (pp) over comparable baselines on macro-F1 scores and +17.5 pp sacreBLEU on generation. Finally, ablation studies confirm that joint training on these four objectives yields synergistic representations that consistently outperform single-task configurations across all scales.
%R 10.18653/2026.findings-emnlp.1877
%U https://aclanthology.org/2026.findings-emnlp.1877/
%U https://doi.org/10.18653/2026.findings-emnlp.1877
%P 40035-40052
Markdown (Informal)
[ARJUN: A Parameter-Efficient Multi-Objective Pre-Training Framework for Language Models](https://aclanthology.org/2026.findings-emnlp.1877/) (Maji & Bhattacharya, Findings 2026)
ACL