@inproceedings{nieth-etal-2026-human,
title = "How Human-Like Are Large Language Models? A Register-Aware Linguistic Evaluation Framework",
author = {Nieth, Bj{\"o}rn and
Gracheva, Marianna and
Mahlberg, Michaela and
Eskofier, Bjoern and
Salin, Emmanuelle},
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.34/",
pages = "715--755",
ISBN = "979-8-89176-459-0",
abstract = "While factual correctness and task-performance have received much attention by Large Language Model (LLM) research, the fundamental question of how human-like generated texts are on a linguistic level has been underexplored. From a corpus-linguistic perspective, language production is inherently context-dependent. Distinct communicative contexts give rise to differences in frequencies and co-occurrence patterns of linguistic features that shape registers of language use. A text failing to adhere to these patterns can be correct content-wise, but still may not appear natural to human readers. In this work, we propose a context-aware evaluation framework in which human-likeness is measured as a two-sample problem between the linguistic feature distribution of a human reference corpus for a given register and a corresponding LLM-generated corpus. We implement this framework using the Maximum Mean Discrepancy (MMD) measure and the 67 lexico-grammatical features introduced by Biber, which are commonly applied in corpus linguistics.\par In our experiments, we compare seven instruction-tuned, open-source models and look at five English-language datasets spanning distinct registers. While across all tested setups, LLMs deviate from the human baseline, which models are closest to human language depends on the register and is not dictated by model size."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="nieth-etal-2026-human">
<titleInfo>
<title>How Human-Like Are Large Language Models? A Register-Aware Linguistic Evaluation Framework</title>
</titleInfo>
<name type="personal">
<namePart type="given">Björn</namePart>
<namePart type="family">Nieth</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marianna</namePart>
<namePart type="family">Gracheva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michaela</namePart>
<namePart type="family">Mahlberg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bjoern</namePart>
<namePart type="family">Eskofier</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Emmanuelle</namePart>
<namePart type="family">Salin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>While factual correctness and task-performance have received much attention by Large Language Model (LLM) research, the fundamental question of how human-like generated texts are on a linguistic level has been underexplored. From a corpus-linguistic perspective, language production is inherently context-dependent. Distinct communicative contexts give rise to differences in frequencies and co-occurrence patterns of linguistic features that shape registers of language use. A text failing to adhere to these patterns can be correct content-wise, but still may not appear natural to human readers. In this work, we propose a context-aware evaluation framework in which human-likeness is measured as a two-sample problem between the linguistic feature distribution of a human reference corpus for a given register and a corresponding LLM-generated corpus. We implement this framework using the Maximum Mean Discrepancy (MMD) measure and the 67 lexico-grammatical features introduced by Biber, which are commonly applied in corpus linguistics.\par In our experiments, we compare seven instruction-tuned, open-source models and look at five English-language datasets spanning distinct registers. While across all tested setups, LLMs deviate from the human baseline, which models are closest to human language depends on the register and is not dictated by model size.</abstract>
<identifier type="citekey">nieth-etal-2026-human</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.34/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>715</start>
<end>755</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T How Human-Like Are Large Language Models? A Register-Aware Linguistic Evaluation Framework
%A Nieth, Björn
%A Gracheva, Marianna
%A Mahlberg, Michaela
%A Eskofier, Bjoern
%A Salin, Emmanuelle
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F nieth-etal-2026-human
%X While factual correctness and task-performance have received much attention by Large Language Model (LLM) research, the fundamental question of how human-like generated texts are on a linguistic level has been underexplored. From a corpus-linguistic perspective, language production is inherently context-dependent. Distinct communicative contexts give rise to differences in frequencies and co-occurrence patterns of linguistic features that shape registers of language use. A text failing to adhere to these patterns can be correct content-wise, but still may not appear natural to human readers. In this work, we propose a context-aware evaluation framework in which human-likeness is measured as a two-sample problem between the linguistic feature distribution of a human reference corpus for a given register and a corresponding LLM-generated corpus. We implement this framework using the Maximum Mean Discrepancy (MMD) measure and the 67 lexico-grammatical features introduced by Biber, which are commonly applied in corpus linguistics.\par In our experiments, we compare seven instruction-tuned, open-source models and look at five English-language datasets spanning distinct registers. While across all tested setups, LLMs deviate from the human baseline, which models are closest to human language depends on the register and is not dictated by model size.
%U https://aclanthology.org/2026.emnlp-main.34/
%P 715-755
Markdown (Informal)
[How Human-Like Are Large Language Models? A Register-Aware Linguistic Evaluation Framework](https://aclanthology.org/2026.emnlp-main.34/) (Nieth et al., EMNLP 2026)
ACL