@inproceedings{catalan-gris-etal-2026-linguists,
title = "The Linguist{'}s Lie Detector: Linguistic Knowledge in Large Language Models",
author = "Catal{\'a}n Gris, Luc{\'i}a and
Gerdes, Kim and
Lee, John S. Y.",
editor = "Rehm, Georg and
Dietze, Stefan and
Dessi, Danilo and
Maynard, Diana and
Schimmler, Sonja",
booktitle = "Proceedings of Natural Scientific Language Processing ({NSLP}) @ {LREC} 2026",
month = may,
year = "2026",
address = "Palma, Mallorca (Spain)",
publisher = "ELRA Language Resources Association (ELRA)",
url = "https://aclanthology.org/2026.nslp-1.23/",
doi = "10.63317/2f7zobe6yo7h",
pages = "235--246",
abstract = "We present a benchmark and evaluation pipeline for assessing how well large language models (LLMs) handle linguistic knowledge. Starting from a curated subcorpus of 11 syntax-focused articles published in Glossa: A Journal of General Linguistics (2016{--}2026), we design a pipeline that (1) segments article text into sentences, (2) extracts atomic, verifiable statements, and (3) classifies them into linguistic categories (language-specific, typological, theoretical, citation, or structural). Each stage is evaluated against human gold annotations produced by three annotators, with inter-annotator agreement measured via Krippendorff{'}s {\ensuremath{\alpha}} and Cohen{'}s {\ensuremath{\kappa}}. We compare several LLMs on extraction and classification, using BERTScore-style similarity for extraction and macro F1 for classification. Finally, we generate contradictions of the true linguistic statements and test whether LLMs can distinguish true from false claims. On a challenge set of 705 linguistic statements, we compare eight LLMs, with Gemini 3 Flash achieving the highest F1 score of 0.66, indicating that current models possess limited but non-trivial linguistic knowledge."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="catalan-gris-etal-2026-linguists">
<titleInfo>
<title>The Linguist’s Lie Detector: Linguistic Knowledge in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lucía</namePart>
<namePart type="family">Catalán Gris</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kim</namePart>
<namePart type="family">Gerdes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">John</namePart>
<namePart type="given">S</namePart>
<namePart type="given">Y</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of Natural Scientific Language Processing (NSLP) @ LREC 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Georg</namePart>
<namePart type="family">Rehm</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Stefan</namePart>
<namePart type="family">Dietze</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Danilo</namePart>
<namePart type="family">Dessi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Diana</namePart>
<namePart type="family">Maynard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sonja</namePart>
<namePart type="family">Schimmler</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resources Association (ELRA)</publisher>
<place>
<placeTerm type="text">Palma, Mallorca (Spain)</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>We present a benchmark and evaluation pipeline for assessing how well large language models (LLMs) handle linguistic knowledge. Starting from a curated subcorpus of 11 syntax-focused articles published in Glossa: A Journal of General Linguistics (2016–2026), we design a pipeline that (1) segments article text into sentences, (2) extracts atomic, verifiable statements, and (3) classifies them into linguistic categories (language-specific, typological, theoretical, citation, or structural). Each stage is evaluated against human gold annotations produced by three annotators, with inter-annotator agreement measured via Krippendorff’s \ensuremathα and Cohen’s \ensuremathąppa. We compare several LLMs on extraction and classification, using BERTScore-style similarity for extraction and macro F1 for classification. Finally, we generate contradictions of the true linguistic statements and test whether LLMs can distinguish true from false claims. On a challenge set of 705 linguistic statements, we compare eight LLMs, with Gemini 3 Flash achieving the highest F1 score of 0.66, indicating that current models possess limited but non-trivial linguistic knowledge.</abstract>
<identifier type="citekey">catalan-gris-etal-2026-linguists</identifier>
<identifier type="doi">10.63317/2f7zobe6yo7h</identifier>
<location>
<url>https://aclanthology.org/2026.nslp-1.23/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>235</start>
<end>246</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Linguist’s Lie Detector: Linguistic Knowledge in Large Language Models
%A Catalán Gris, Lucía
%A Gerdes, Kim
%A Lee, John S. Y.
%Y Rehm, Georg
%Y Dietze, Stefan
%Y Dessi, Danilo
%Y Maynard, Diana
%Y Schimmler, Sonja
%S Proceedings of Natural Scientific Language Processing (NSLP) @ LREC 2026
%D 2026
%8 May
%I ELRA Language Resources Association (ELRA)
%C Palma, Mallorca (Spain)
%F catalan-gris-etal-2026-linguists
%X We present a benchmark and evaluation pipeline for assessing how well large language models (LLMs) handle linguistic knowledge. Starting from a curated subcorpus of 11 syntax-focused articles published in Glossa: A Journal of General Linguistics (2016–2026), we design a pipeline that (1) segments article text into sentences, (2) extracts atomic, verifiable statements, and (3) classifies them into linguistic categories (language-specific, typological, theoretical, citation, or structural). Each stage is evaluated against human gold annotations produced by three annotators, with inter-annotator agreement measured via Krippendorff’s \ensuremathα and Cohen’s \ensuremathąppa. We compare several LLMs on extraction and classification, using BERTScore-style similarity for extraction and macro F1 for classification. Finally, we generate contradictions of the true linguistic statements and test whether LLMs can distinguish true from false claims. On a challenge set of 705 linguistic statements, we compare eight LLMs, with Gemini 3 Flash achieving the highest F1 score of 0.66, indicating that current models possess limited but non-trivial linguistic knowledge.
%R 10.63317/2f7zobe6yo7h
%U https://aclanthology.org/2026.nslp-1.23/
%U https://doi.org/10.63317/2f7zobe6yo7h
%P 235-246
Markdown (Informal)
[The Linguist’s Lie Detector: Linguistic Knowledge in Large Language Models](https://aclanthology.org/2026.nslp-1.23/) (Catalán Gris et al., NSLP 2026)
ACL