@inproceedings{suarez-etal-2026-assessing,
title = "Assessing the Comprehension of Regional Colombian Idioms by Language Models",
author = "Suarez, Santiago Bobadilla and
Osorio, Juan Diego and
Roncancio, Maria Alejandra Pinzon and
Chaparro, Ignacio and
Manrique, Ruben",
editor = "Sousa-Silva, Rui and
Lopes Cardoso, Henrique and
Koponen, Maarit and
Pareja-Lora, Antonio",
booktitle = "Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era ({LUHME} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.luhme-1.14/",
pages = "173--182",
ISBN = "979-8-89176-471-2",
abstract = "This study queries 23 large language models (LLMs) and reports analyses for 22 models on Colombian idioms and regionally marked expressions. We introduce an evaluation set derived from Colombian lexicographical sources and assess models on three tasks: positive-class idiom recognition, definition generation, and context-based paraphrase generation. Automatic metrics show substantial variation across models, with Gemini-3-Flash-Preview achieving the highest average score on the meaning-oriented tasks. Human evaluation of four selected models qualifies these rankings: the best Task 2 definition accuracy reached only 58.57{\%}, while Task 3 accuracy was higher for all four models. Exploratory regional analysis of the tagged subset shows uneven performance across Colombian macro-regions, with lower results for expressions from the Llanos and high Caribbean regions. These findings suggest that current LLMs only partially capture Colombian idiomatic meaning and motivate more regionally balanced resources and evaluation protocols for Spanish."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="suarez-etal-2026-assessing">
<titleInfo>
<title>Assessing the Comprehension of Regional Colombian Idioms by Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Santiago</namePart>
<namePart type="given">Bobadilla</namePart>
<namePart type="family">Suarez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juan</namePart>
<namePart type="given">Diego</namePart>
<namePart type="family">Osorio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="given">Alejandra</namePart>
<namePart type="given">Pinzon</namePart>
<namePart type="family">Roncancio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ignacio</namePart>
<namePart type="family">Chaparro</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruben</namePart>
<namePart type="family">Manrique</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era (LUHME 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rui</namePart>
<namePart type="family">Sousa-Silva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henrique</namePart>
<namePart type="family">Lopes Cardoso</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maarit</namePart>
<namePart type="family">Koponen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Pareja-Lora</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-471-2</identifier>
</relatedItem>
<abstract>This study queries 23 large language models (LLMs) and reports analyses for 22 models on Colombian idioms and regionally marked expressions. We introduce an evaluation set derived from Colombian lexicographical sources and assess models on three tasks: positive-class idiom recognition, definition generation, and context-based paraphrase generation. Automatic metrics show substantial variation across models, with Gemini-3-Flash-Preview achieving the highest average score on the meaning-oriented tasks. Human evaluation of four selected models qualifies these rankings: the best Task 2 definition accuracy reached only 58.57%, while Task 3 accuracy was higher for all four models. Exploratory regional analysis of the tagged subset shows uneven performance across Colombian macro-regions, with lower results for expressions from the Llanos and high Caribbean regions. These findings suggest that current LLMs only partially capture Colombian idiomatic meaning and motivate more regionally balanced resources and evaluation protocols for Spanish.</abstract>
<identifier type="citekey">suarez-etal-2026-assessing</identifier>
<location>
<url>https://aclanthology.org/2026.luhme-1.14/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>173</start>
<end>182</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Assessing the Comprehension of Regional Colombian Idioms by Language Models
%A Suarez, Santiago Bobadilla
%A Osorio, Juan Diego
%A Roncancio, Maria Alejandra Pinzon
%A Chaparro, Ignacio
%A Manrique, Ruben
%Y Sousa-Silva, Rui
%Y Lopes Cardoso, Henrique
%Y Koponen, Maarit
%Y Pareja-Lora, Antonio
%S Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era (LUHME 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-471-2
%F suarez-etal-2026-assessing
%X This study queries 23 large language models (LLMs) and reports analyses for 22 models on Colombian idioms and regionally marked expressions. We introduce an evaluation set derived from Colombian lexicographical sources and assess models on three tasks: positive-class idiom recognition, definition generation, and context-based paraphrase generation. Automatic metrics show substantial variation across models, with Gemini-3-Flash-Preview achieving the highest average score on the meaning-oriented tasks. Human evaluation of four selected models qualifies these rankings: the best Task 2 definition accuracy reached only 58.57%, while Task 3 accuracy was higher for all four models. Exploratory regional analysis of the tagged subset shows uneven performance across Colombian macro-regions, with lower results for expressions from the Llanos and high Caribbean regions. These findings suggest that current LLMs only partially capture Colombian idiomatic meaning and motivate more regionally balanced resources and evaluation protocols for Spanish.
%U https://aclanthology.org/2026.luhme-1.14/
%P 173-182
Markdown (Informal)
[Assessing the Comprehension of Regional Colombian Idioms by Language Models](https://aclanthology.org/2026.luhme-1.14/) (Suarez et al., LUHME 2026)
ACL
- Santiago Bobadilla Suarez, Juan Diego Osorio, Maria Alejandra Pinzon Roncancio, Ignacio Chaparro, and Ruben Manrique. 2026. Assessing the Comprehension of Regional Colombian Idioms by Language Models. In Proceedings of the 3rd Workshop on Language Understanding in the Human-Machine Era (LUHME 2026), pages 173–182, Budapest, Hungary. Association for Computational Linguistics.