@inproceedings{canaverde-etal-2026-legalbench,
title = "{L}egal{B}ench.{PT}: A Benchmark for {P}ortuguese Law",
author = "Canaverde, Beatriz and
Pires, Telmo and
Martins, Andre",
editor = "Aletras, Nikolaos and
Chalkidis, Ilias and
Barrett, Leslie and
Goanț{\u{a}}, C{\u{a}}t{\u{a}}lina and
Preoțiuc-Pietro, Daniel and
Spanakis, Gerasimos and
Xu, Shanshan",
booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.nllp-1.30/",
pages = "334--349",
ISBN = "979-8-89176-494-1",
abstract = "The growing use of LLMs in the legal field has spurred the creation of benchmarks across various jurisdictions and languages, yet none specifically focused on the Portuguese legal system. We present LegalBench.PT, the first comprehensive benchmark for Portuguese law. To develop it, we first collect long-form questions and answers from real law exams, and then use GPT-4o to convert them into multiple-choice, true/false, and matching formats. Once generated, the questions are filtered and processed to improve the quality of the dataset. To ensure accuracy and relevance, we validate our approach by having a legal professional review a sample of the generated questions. Although synthetically generated, we show that their basis in human-created exams and our rigorous filtering and processing methods applied result in a reliable benchmark for assessing LLMs' legal knowledge and reasoning abilities. Finally, we evaluate the performance of leading LLMs on LegalBench.PT and investigate potential biases in GPT-4o{'}s responses. We also assess the performance of Portuguese lawyers on a sample of questions to establish a baseline for model comparison and validate the benchmark."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="canaverde-etal-2026-legalbench">
<titleInfo>
<title>LegalBench.PT: A Benchmark for Portuguese Law</title>
</titleInfo>
<name type="personal">
<namePart type="given">Beatriz</namePart>
<namePart type="family">Canaverde</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Telmo</namePart>
<namePart type="family">Pires</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andre</namePart>
<namePart type="family">Martins</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Natural Legal Language Processing Workshop 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ilias</namePart>
<namePart type="family">Chalkidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leslie</namePart>
<namePart type="family">Barrett</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Cătălina</namePart>
<namePart type="family">Goanță</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Daniel</namePart>
<namePart type="family">Preoțiuc-Pietro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gerasimos</namePart>
<namePart type="family">Spanakis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shanshan</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-494-1</identifier>
</relatedItem>
<abstract>The growing use of LLMs in the legal field has spurred the creation of benchmarks across various jurisdictions and languages, yet none specifically focused on the Portuguese legal system. We present LegalBench.PT, the first comprehensive benchmark for Portuguese law. To develop it, we first collect long-form questions and answers from real law exams, and then use GPT-4o to convert them into multiple-choice, true/false, and matching formats. Once generated, the questions are filtered and processed to improve the quality of the dataset. To ensure accuracy and relevance, we validate our approach by having a legal professional review a sample of the generated questions. Although synthetically generated, we show that their basis in human-created exams and our rigorous filtering and processing methods applied result in a reliable benchmark for assessing LLMs’ legal knowledge and reasoning abilities. Finally, we evaluate the performance of leading LLMs on LegalBench.PT and investigate potential biases in GPT-4o’s responses. We also assess the performance of Portuguese lawyers on a sample of questions to establish a baseline for model comparison and validate the benchmark.</abstract>
<identifier type="citekey">canaverde-etal-2026-legalbench</identifier>
<location>
<url>https://aclanthology.org/2026.nllp-1.30/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>334</start>
<end>349</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T LegalBench.PT: A Benchmark for Portuguese Law
%A Canaverde, Beatriz
%A Pires, Telmo
%A Martins, Andre
%Y Aletras, Nikolaos
%Y Chalkidis, Ilias
%Y Barrett, Leslie
%Y Goanță, Cătălina
%Y Preoțiuc-Pietro, Daniel
%Y Spanakis, Gerasimos
%Y Xu, Shanshan
%S Proceedings of the Natural Legal Language Processing Workshop 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-494-1
%F canaverde-etal-2026-legalbench
%X The growing use of LLMs in the legal field has spurred the creation of benchmarks across various jurisdictions and languages, yet none specifically focused on the Portuguese legal system. We present LegalBench.PT, the first comprehensive benchmark for Portuguese law. To develop it, we first collect long-form questions and answers from real law exams, and then use GPT-4o to convert them into multiple-choice, true/false, and matching formats. Once generated, the questions are filtered and processed to improve the quality of the dataset. To ensure accuracy and relevance, we validate our approach by having a legal professional review a sample of the generated questions. Although synthetically generated, we show that their basis in human-created exams and our rigorous filtering and processing methods applied result in a reliable benchmark for assessing LLMs’ legal knowledge and reasoning abilities. Finally, we evaluate the performance of leading LLMs on LegalBench.PT and investigate potential biases in GPT-4o’s responses. We also assess the performance of Portuguese lawyers on a sample of questions to establish a baseline for model comparison and validate the benchmark.
%U https://aclanthology.org/2026.nllp-1.30/
%P 334-349
Markdown (Informal)
[LegalBench.PT: A Benchmark for Portuguese Law](https://aclanthology.org/2026.nllp-1.30/) (Canaverde et al., NLLP 2026)
ACL
- Beatriz Canaverde, Telmo Pires, and Andre Martins. 2026. LegalBench.PT: A Benchmark for Portuguese Law. In Proceedings of the Natural Legal Language Processing Workshop 2026, pages 334–349, Budapest, Hungary. Association for Computational Linguistics.