@inproceedings{mahadi-etal-2026-small,
title = "Do Small Models Use the Law You Give Them? Measuring Context Use on a Bilingual {B}angladesh Legal Benchmark",
author = "Mahadi, Moniruzzaman and
Alam, Abrar Mohammed Tanzim and
Abdullah, Mir Mohammad Asif and
Monalisa, Sayma Siddika and
Shatabda, Swakkhar and
Arefeen, Md Adnan",
editor = "Aletras, Nikolaos and
Chalkidis, Ilias and
Barrett, Leslie and
Goanț{\u{a}}, C{\u{a}}t{\u{a}}lina and
Preoțiuc-Pietro, Daniel and
Spanakis, Gerasimos and
Xu, Shanshan",
booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nllp-1.24/",
pages = "271--289",
ISBN = "979-8-89176-494-1",
abstract = "Fine-tuning improves measured accuracy on legal question answering, but does it improve how models use supplied law? We investigate this question in bilingual Bangladeshi legal QA, distinguishing gains due to scoring and retrieval from changes in reliance on the governing provision. We adapt six small instruction-tuned models with LoRA across three random seeds and evaluate them using constrained option-letter scoring, cyclic option rotation, and matched contexts that include or omit the governing provision. Scoring choices substantially change the apparent benefit of adaptation: a 50.0{\%} gain under exact-line parsing falls to 3.0{\%} under option scoring, and, for one model, the two methods reverse the ranking of systems. Supplied law improves accuracy, and removing the governing provision degrades performance for both base and adapted models. However, difference-in-differences estimates detect no increase in reliance on that provision after fine-tuning. These findings identify a limit to interpreting accuracy gains as evidence of legal adaptation: improved performance need not imply improved use of legal context. Evaluations must therefore distinguish scorer, retriever, and model effects. We release a hierarchy-preserving statutory corpus, 2,165 reviewed bilingual training examples, a 150-item supplied-law benchmark, and evaluation code at \url{https://github.com/m-mahadi/bangladesh-legal-qa}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mahadi-etal-2026-small">
<titleInfo>
<title>Do Small Models Use the Law You Give Them? Measuring Context Use on a Bilingual Bangladesh Legal Benchmark</title>
</titleInfo>
<name type="personal">
<namePart type="given">Moniruzzaman</namePart>
<namePart type="family">Mahadi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abrar</namePart>
<namePart type="given">Mohammed</namePart>
<namePart type="given">Tanzim</namePart>
<namePart type="family">Alam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mir</namePart>
<namePart type="given">Mohammad</namePart>
<namePart type="given">Asif</namePart>
<namePart type="family">Abdullah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sayma</namePart>
<namePart type="given">Siddika</namePart>
<namePart type="family">Monalisa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Swakkhar</namePart>
<namePart type="family">Shatabda</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Adnan</namePart>
<namePart type="family">Arefeen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Natural Legal Language Processing Workshop 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Chalkidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leslie</namePart>
<namePart type="family">Barrett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cătălina</namePart>
<namePart type="family">Goanță</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoțiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Spanakis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-494-1</identifier>
</relatedItem>
<abstract>Fine-tuning improves measured accuracy on legal question answering, but does it improve how models use supplied law? We investigate this question in bilingual Bangladeshi legal QA, distinguishing gains due to scoring and retrieval from changes in reliance on the governing provision. We adapt six small instruction-tuned models with LoRA across three random seeds and evaluate them using constrained option-letter scoring, cyclic option rotation, and matched contexts that include or omit the governing provision. Scoring choices substantially change the apparent benefit of adaptation: a 50.0% gain under exact-line parsing falls to 3.0% under option scoring, and, for one model, the two methods reverse the ranking of systems. Supplied law improves accuracy, and removing the governing provision degrades performance for both base and adapted models. However, difference-in-differences estimates detect no increase in reliance on that provision after fine-tuning. These findings identify a limit to interpreting accuracy gains as evidence of legal adaptation: improved performance need not imply improved use of legal context. Evaluations must therefore distinguish scorer, retriever, and model effects. We release a hierarchy-preserving statutory corpus, 2,165 reviewed bilingual training examples, a 150-item supplied-law benchmark, and evaluation code at https://github.com/m-mahadi/bangladesh-legal-qa.</abstract>
<identifier type="citekey">mahadi-etal-2026-small</identifier>
<location>
<url>https://aclanthology.org/2026.nllp-1.24/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>271</start>
<end>289</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Do Small Models Use the Law You Give Them? Measuring Context Use on a Bilingual Bangladesh Legal Benchmark
%A Mahadi, Moniruzzaman
%A Alam, Abrar Mohammed Tanzim
%A Abdullah, Mir Mohammad Asif
%A Monalisa, Sayma Siddika
%A Shatabda, Swakkhar
%A Arefeen, Md Adnan
%Y Aletras, Nikolaos
%Y Chalkidis, Ilias
%Y Barrett, Leslie
%Y Goanță, Cătălina
%Y Preoțiuc-Pietro, Daniel
%Y Spanakis, Gerasimos
%Y Xu, Shanshan
%S Proceedings of the Natural Legal Language Processing Workshop 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-494-1
%F mahadi-etal-2026-small
%X Fine-tuning improves measured accuracy on legal question answering, but does it improve how models use supplied law? We investigate this question in bilingual Bangladeshi legal QA, distinguishing gains due to scoring and retrieval from changes in reliance on the governing provision. We adapt six small instruction-tuned models with LoRA across three random seeds and evaluate them using constrained option-letter scoring, cyclic option rotation, and matched contexts that include or omit the governing provision. Scoring choices substantially change the apparent benefit of adaptation: a 50.0% gain under exact-line parsing falls to 3.0% under option scoring, and, for one model, the two methods reverse the ranking of systems. Supplied law improves accuracy, and removing the governing provision degrades performance for both base and adapted models. However, difference-in-differences estimates detect no increase in reliance on that provision after fine-tuning. These findings identify a limit to interpreting accuracy gains as evidence of legal adaptation: improved performance need not imply improved use of legal context. Evaluations must therefore distinguish scorer, retriever, and model effects. We release a hierarchy-preserving statutory corpus, 2,165 reviewed bilingual training examples, a 150-item supplied-law benchmark, and evaluation code at https://github.com/m-mahadi/bangladesh-legal-qa.
%U https://aclanthology.org/2026.nllp-1.24/
%P 271-289
Markdown (Informal)
[Do Small Models Use the Law You Give Them? Measuring Context Use on a Bilingual Bangladesh Legal Benchmark](https://aclanthology.org/2026.nllp-1.24/) (Mahadi et al., NLLP 2026)
ACL