@inproceedings{gama-etal-2026-hunebr,
title = "{H}u{N}e{BR}: A Multitask Benchmark to Evaluate {LLM}s' Understanding of {N}ortheastern {B}razilian {P}ortuguese Humor",
author = "Gama, Jos{\'e} Matheus do Nascimento and
Maia, David Candeia and
Balby Marinho, Leandro and
Morais, Fabio and
Brunet, Jo{\~a}o",
editor = "Ojha, Atul Kr. and
Sakti, Sakriani and
Soria, Claudia and
Melero, Maite and
McCrae, John P. and
Lignos, Constantine and
Liu, Chao-Hong and
Claramunt, German Rigau and
Rehm, Georg",
booktitle = "Proceedings of the {SIGUL} 2026 Joint Workshop with {ELE}, {EURALI}, and {DCLRL}: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages",
month = may,
year = "2026",
address = "Palma, Mallorca, Spain",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.sigul-1.30/",
doi = "10.63317/22b3hhj77udn",
pages = "299--311",
abstract = "Humor recognition is a major challenge in Natural Language Processing (NLP) due to its subtle and context-dependent nature. Despite advances, Large Language Models (LLMs) still struggle with this task, especially in Brazilian Portuguese, where no dedicated benchmarks exist. This paper presents HuNeBR, a new benchmark of 475 annotated humorous texts from Northeastern Brazilian comedians. The benchmark evaluates LLMs on three tasks: identifying punchlines, classifying texts into eight comic styles, and explaining humor. This is the first benchmark to evaluate LLMs on the in-depth interpretation of humorous texts in Brazilian Portuguese, going beyond the binary tasks of traditional humor benchmarks. Both general-purpose and Portuguese-specialized LLMs were evaluated under zero-shot and few-shot settings. The findings indicate that LLMs perform very well at identifying punchlines, show inconsistent results in classifying comic styles, and produce humor interpretations that mostly align with human judgments. Among the models assessed, general-purpose multilingual systems like GPT-4 and Gemini 2.5 Flash achieved the top overall performance, whereas Sabi{\'a} 3.1, a model specialized in Brazilian Portuguese, demonstrated competitive results across all three tasks, highlighting the value of locally trained models in capturing linguistic and cultural subtleties."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="gama-etal-2026-hunebr">
<titleInfo>
<title>HuNeBR: A Multitask Benchmark to Evaluate LLMs’ Understanding of Northeastern Brazilian Portuguese Humor</title>
</titleInfo>
<name type="personal">
<namePart type="given">José</namePart>
<namePart type="given">Matheus</namePart>
<namePart type="given">do</namePart>
<namePart type="given">Nascimento</namePart>
<namePart type="family">Gama</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="given">Candeia</namePart>
<namePart type="family">Maia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leandro</namePart>
<namePart type="family">Balby Marinho</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Fabio</namePart>
<namePart type="family">Morais</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">João</namePart>
<namePart type="family">Brunet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the SIGUL 2026 Joint Workshop with ELE, EURALI, and DCLRL: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages</title>
</titleInfo>
<name type="personal">
<namePart type="given">Atul</namePart>
<namePart type="given">Kr.</namePart>
<namePart type="family">Ojha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakriani</namePart>
<namePart type="family">Sakti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Claudia</namePart>
<namePart type="family">Soria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maite</namePart>
<namePart type="family">Melero</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">John</namePart>
<namePart type="given">P</namePart>
<namePart type="family">McCrae</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Constantine</namePart>
<namePart type="family">Lignos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao-Hong</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">German</namePart>
<namePart type="given">Rigau</namePart>
<namePart type="family">Claramunt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Georg</namePart>
<namePart type="family">Rehm</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Humor recognition is a major challenge in Natural Language Processing (NLP) due to its subtle and context-dependent nature. Despite advances, Large Language Models (LLMs) still struggle with this task, especially in Brazilian Portuguese, where no dedicated benchmarks exist. This paper presents HuNeBR, a new benchmark of 475 annotated humorous texts from Northeastern Brazilian comedians. The benchmark evaluates LLMs on three tasks: identifying punchlines, classifying texts into eight comic styles, and explaining humor. This is the first benchmark to evaluate LLMs on the in-depth interpretation of humorous texts in Brazilian Portuguese, going beyond the binary tasks of traditional humor benchmarks. Both general-purpose and Portuguese-specialized LLMs were evaluated under zero-shot and few-shot settings. The findings indicate that LLMs perform very well at identifying punchlines, show inconsistent results in classifying comic styles, and produce humor interpretations that mostly align with human judgments. Among the models assessed, general-purpose multilingual systems like GPT-4 and Gemini 2.5 Flash achieved the top overall performance, whereas Sabiá 3.1, a model specialized in Brazilian Portuguese, demonstrated competitive results across all three tasks, highlighting the value of locally trained models in capturing linguistic and cultural subtleties.</abstract>
<identifier type="citekey">gama-etal-2026-hunebr</identifier>
<identifier type="doi">10.63317/22b3hhj77udn</identifier>
<location>
<url>https://aclanthology.org/2026.sigul-1.30/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>299</start>
<end>311</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T HuNeBR: A Multitask Benchmark to Evaluate LLMs’ Understanding of Northeastern Brazilian Portuguese Humor
%A Gama, José Matheus do Nascimento
%A Maia, David Candeia
%A Balby Marinho, Leandro
%A Morais, Fabio
%A Brunet, João
%Y Ojha, Atul Kr.
%Y Sakti, Sakriani
%Y Soria, Claudia
%Y Melero, Maite
%Y McCrae, John P.
%Y Lignos, Constantine
%Y Liu, Chao-Hong
%Y Claramunt, German Rigau
%Y Rehm, Georg
%S Proceedings of the SIGUL 2026 Joint Workshop with ELE, EURALI, and DCLRL: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca, Spain
%F gama-etal-2026-hunebr
%X Humor recognition is a major challenge in Natural Language Processing (NLP) due to its subtle and context-dependent nature. Despite advances, Large Language Models (LLMs) still struggle with this task, especially in Brazilian Portuguese, where no dedicated benchmarks exist. This paper presents HuNeBR, a new benchmark of 475 annotated humorous texts from Northeastern Brazilian comedians. The benchmark evaluates LLMs on three tasks: identifying punchlines, classifying texts into eight comic styles, and explaining humor. This is the first benchmark to evaluate LLMs on the in-depth interpretation of humorous texts in Brazilian Portuguese, going beyond the binary tasks of traditional humor benchmarks. Both general-purpose and Portuguese-specialized LLMs were evaluated under zero-shot and few-shot settings. The findings indicate that LLMs perform very well at identifying punchlines, show inconsistent results in classifying comic styles, and produce humor interpretations that mostly align with human judgments. Among the models assessed, general-purpose multilingual systems like GPT-4 and Gemini 2.5 Flash achieved the top overall performance, whereas Sabiá 3.1, a model specialized in Brazilian Portuguese, demonstrated competitive results across all three tasks, highlighting the value of locally trained models in capturing linguistic and cultural subtleties.
%R 10.63317/22b3hhj77udn
%U https://aclanthology.org/2026.sigul-1.30/
%U https://doi.org/10.63317/22b3hhj77udn
%P 299-311
Markdown (Informal)
[HuNeBR: A Multitask Benchmark to Evaluate LLMs’ Understanding of Northeastern Brazilian Portuguese Humor](https://aclanthology.org/2026.sigul-1.30/) (Gama et al., SIGUL-EURALI-DCLRL 2026)
ACL
- José Matheus do Nascimento Gama, David Candeia Maia, Leandro Balby Marinho, Fabio Morais, and João Brunet. 2026. HuNeBR: A Multitask Benchmark to Evaluate LLMs’ Understanding of Northeastern Brazilian Portuguese Humor. In Proceedings of the SIGUL 2026 Joint Workshop with ELE, EURALI, and DCLRL: Towards Inclusivity and Equality: Language Resources and Technologies for Under-Resourced and Endangered Languages, pages 299–311, Palma, Mallorca, Spain. ELRA Language Resources Association (ELRA).