@inproceedings{shanbhag-etal-2025-non,
title = "Non-Contextual {BERT} or {F}ast{T}ext? A Comparative Analysis",
author = "Shanbhag, Abhay and
Jadhav, Suramya and
Thakurdesai, Amogh and
Sinare, Ridhima Bhaskar and
Joshi, Raviraj",
editor = "Das, Sudhansu Bala and
Mishra, Pruthwik and
Singh, Alok and
Muhammad, Shamsuddeen Hassan and
Ekbal, Asif and
Das, Uday Kumar",
booktitle = "Proceedings of the Workshop on Beyond English: Natural Language Processing for all Languages in an Era of Large Language Models",
month = sep,
year = "2025",
address = "Varna, Bulgaria",
publisher = "INCOMA Ltd., Shoumen, BULGARIA",
url = "https://aclanthology.org/2025.globalnlp-1.4/",
pages = "27--33",
abstract = "Natural Language Processing (NLP) for low-resource languages, which lack large annotated datasets, faces significant challenges due to limited high-quality data and linguistic resources. The selection of embeddings plays a critical role in achieving strong performance in NLP tasks. While contextual BERT embeddings require a full forward pass, non-contextual BERT embeddings rely only on table lookup. Existing research has primarily focused on contextual BERT embeddings, leaving non-contextual embeddings largely unexplored. In this study, we analyze the effectiveness of non-contextual embeddings from BERT models (MuRIL and MahaBERT) and FastText models (IndicFT and MahaFT) for tasks such as news classification, sentiment analysis, and hate speech detection in one such low-resource language{---}Marathi. We compare these embeddings with their contextual and compressed variants. Our findings indicate that non-contextual BERT embeddings extracted from the model{'}s first embedding layer outperform FastText embeddings, presenting a promising alternative for low-resource NLP."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shanbhag-etal-2025-non">
<titleInfo>
<title>Non-Contextual BERT or FastText? A Comparative Analysis</title>
</titleInfo>
<name type="personal">
<namePart type="given">Abhay</namePart>
<namePart type="family">Shanbhag</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suramya</namePart>
<namePart type="family">Jadhav</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Amogh</namePart>
<namePart type="family">Thakurdesai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ridhima</namePart>
<namePart type="given">Bhaskar</namePart>
<namePart type="family">Sinare</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raviraj</namePart>
<namePart type="family">Joshi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-09</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Workshop on Beyond English: Natural Language Processing for all Languages in an Era of Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sudhansu</namePart>
<namePart type="given">Bala</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pruthwik</namePart>
<namePart type="family">Mishra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alok</namePart>
<namePart type="family">Singh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shamsuddeen</namePart>
<namePart type="given">Hassan</namePart>
<namePart type="family">Muhammad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Asif</namePart>
<namePart type="family">Ekbal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Uday</namePart>
<namePart type="given">Kumar</namePart>
<namePart type="family">Das</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>INCOMA Ltd., Shoumen, BULGARIA</publisher>
<place>
<placeTerm type="text">Varna, Bulgaria</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Natural Language Processing (NLP) for low-resource languages, which lack large annotated datasets, faces significant challenges due to limited high-quality data and linguistic resources. The selection of embeddings plays a critical role in achieving strong performance in NLP tasks. While contextual BERT embeddings require a full forward pass, non-contextual BERT embeddings rely only on table lookup. Existing research has primarily focused on contextual BERT embeddings, leaving non-contextual embeddings largely unexplored. In this study, we analyze the effectiveness of non-contextual embeddings from BERT models (MuRIL and MahaBERT) and FastText models (IndicFT and MahaFT) for tasks such as news classification, sentiment analysis, and hate speech detection in one such low-resource language—Marathi. We compare these embeddings with their contextual and compressed variants. Our findings indicate that non-contextual BERT embeddings extracted from the model’s first embedding layer outperform FastText embeddings, presenting a promising alternative for low-resource NLP.</abstract>
<identifier type="citekey">shanbhag-etal-2025-non</identifier>
<location>
<url>https://aclanthology.org/2025.globalnlp-1.4/</url>
</location>
<part>
<date>2025-09</date>
<extent unit="page">
<start>27</start>
<end>33</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Non-Contextual BERT or FastText? A Comparative Analysis
%A Shanbhag, Abhay
%A Jadhav, Suramya
%A Thakurdesai, Amogh
%A Sinare, Ridhima Bhaskar
%A Joshi, Raviraj
%Y Das, Sudhansu Bala
%Y Mishra, Pruthwik
%Y Singh, Alok
%Y Muhammad, Shamsuddeen Hassan
%Y Ekbal, Asif
%Y Das, Uday Kumar
%S Proceedings of the Workshop on Beyond English: Natural Language Processing for all Languages in an Era of Large Language Models
%D 2025
%8 September
%I INCOMA Ltd., Shoumen, BULGARIA
%C Varna, Bulgaria
%F shanbhag-etal-2025-non
%X Natural Language Processing (NLP) for low-resource languages, which lack large annotated datasets, faces significant challenges due to limited high-quality data and linguistic resources. The selection of embeddings plays a critical role in achieving strong performance in NLP tasks. While contextual BERT embeddings require a full forward pass, non-contextual BERT embeddings rely only on table lookup. Existing research has primarily focused on contextual BERT embeddings, leaving non-contextual embeddings largely unexplored. In this study, we analyze the effectiveness of non-contextual embeddings from BERT models (MuRIL and MahaBERT) and FastText models (IndicFT and MahaFT) for tasks such as news classification, sentiment analysis, and hate speech detection in one such low-resource language—Marathi. We compare these embeddings with their contextual and compressed variants. Our findings indicate that non-contextual BERT embeddings extracted from the model’s first embedding layer outperform FastText embeddings, presenting a promising alternative for low-resource NLP.
%U https://aclanthology.org/2025.globalnlp-1.4/
%P 27-33
Markdown (Informal)
[Non-Contextual BERT or FastText? A Comparative Analysis](https://aclanthology.org/2025.globalnlp-1.4/) (Shanbhag et al., GlobalNLP 2025)
ACL
- Abhay Shanbhag, Suramya Jadhav, Amogh Thakurdesai, Ridhima Bhaskar Sinare, and Raviraj Joshi. 2025. Non-Contextual BERT or FastText? A Comparative Analysis. In Proceedings of the Workshop on Beyond English: Natural Language Processing for all Languages in an Era of Large Language Models, pages 27–33, Varna, Bulgaria. INCOMA Ltd., Shoumen, BULGARIA.