@inproceedings{maurer-etal-2026-promptology,
title = "Promptology: A Large-Scale Systematic Analysis of Prompt Elements and Contextual Factors in Argument Generation",
author = "Maurer, Maximilian and
Cotovio, Ana Lisboa and
Melis, Matteo and
Romberg, Julia and
Costa, Aldo and
Lapesa, Gabriella",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.428/",
pages = "9794--9826",
ISBN = "979-8-89176-459-0",
abstract = "The ``political opinions'' of Large Language Models (LLMs) increasingly find their way into political discourse despite limited under- standing and control of these systems. Our paper adds to the understanding of how prompt manipulation impacts the multilingual generation of political arguments, focusing on two fundamental aspects of generated text: style and semantics. Starting from a reference set of Swiss politicians' arguments in German, Italian, and French, we conduct a large-scale prompt-manipulation experiment on 4 LLMs, building the prompts in a modular way (i.e., with blocks encoding socio-demographics, context, and examples) and testing all possible combinations. Using our parallel dataset of over 375k human argument-aligned generated arguments, we assess the detectability of LLM arguments through classification experiments with 160 linguistic features and semantic embeddings. We then carry out quantitative and qualitative analysis to understand how human- written vs. LLM-generated text compares, and to assess the impact of prompt manipulation.\par Our results show that human and LLM arguments are substantially different in style and semantics, making their identification easy for classifiers. Prompt manipulation does not substantially impact semantics and style, except for sociodemographic persona prompting, which makes generated texts moderately more human-like. Model, language, and topics impact the stability of the generated text most. We make the code, data, and raw results available via https://github.com/mmmaurer/promptology."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="maurer-etal-2026-promptology">
<titleInfo>
<title>Promptology: A Large-Scale Systematic Analysis of Prompt Elements and Contextual Factors in Argument Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maximilian</namePart>
<namePart type="family">Maurer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ana</namePart>
<namePart type="given">Lisboa</namePart>
<namePart type="family">Cotovio</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Matteo</namePart>
<namePart type="family">Melis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julia</namePart>
<namePart type="family">Romberg</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aldo</namePart>
<namePart type="family">Costa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriella</namePart>
<namePart type="family">Lapesa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>The “political opinions” of Large Language Models (LLMs) increasingly find their way into political discourse despite limited under- standing and control of these systems. Our paper adds to the understanding of how prompt manipulation impacts the multilingual generation of political arguments, focusing on two fundamental aspects of generated text: style and semantics. Starting from a reference set of Swiss politicians’ arguments in German, Italian, and French, we conduct a large-scale prompt-manipulation experiment on 4 LLMs, building the prompts in a modular way (i.e., with blocks encoding socio-demographics, context, and examples) and testing all possible combinations. Using our parallel dataset of over 375k human argument-aligned generated arguments, we assess the detectability of LLM arguments through classification experiments with 160 linguistic features and semantic embeddings. We then carry out quantitative and qualitative analysis to understand how human- written vs. LLM-generated text compares, and to assess the impact of prompt manipulation.\par Our results show that human and LLM arguments are substantially different in style and semantics, making their identification easy for classifiers. Prompt manipulation does not substantially impact semantics and style, except for sociodemographic persona prompting, which makes generated texts moderately more human-like. Model, language, and topics impact the stability of the generated text most. We make the code, data, and raw results available via https://github.com/mmmaurer/promptology.</abstract>
<identifier type="citekey">maurer-etal-2026-promptology</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.428/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>9794</start>
<end>9826</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Promptology: A Large-Scale Systematic Analysis of Prompt Elements and Contextual Factors in Argument Generation
%A Maurer, Maximilian
%A Cotovio, Ana Lisboa
%A Melis, Matteo
%A Romberg, Julia
%A Costa, Aldo
%A Lapesa, Gabriella
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F maurer-etal-2026-promptology
%X The “political opinions” of Large Language Models (LLMs) increasingly find their way into political discourse despite limited under- standing and control of these systems. Our paper adds to the understanding of how prompt manipulation impacts the multilingual generation of political arguments, focusing on two fundamental aspects of generated text: style and semantics. Starting from a reference set of Swiss politicians’ arguments in German, Italian, and French, we conduct a large-scale prompt-manipulation experiment on 4 LLMs, building the prompts in a modular way (i.e., with blocks encoding socio-demographics, context, and examples) and testing all possible combinations. Using our parallel dataset of over 375k human argument-aligned generated arguments, we assess the detectability of LLM arguments through classification experiments with 160 linguistic features and semantic embeddings. We then carry out quantitative and qualitative analysis to understand how human- written vs. LLM-generated text compares, and to assess the impact of prompt manipulation.\par Our results show that human and LLM arguments are substantially different in style and semantics, making their identification easy for classifiers. Prompt manipulation does not substantially impact semantics and style, except for sociodemographic persona prompting, which makes generated texts moderately more human-like. Model, language, and topics impact the stability of the generated text most. We make the code, data, and raw results available via https://github.com/mmmaurer/promptology.
%U https://aclanthology.org/2026.emnlp-main.428/
%P 9794-9826
Markdown (Informal)
[Promptology: A Large-Scale Systematic Analysis of Prompt Elements and Contextual Factors in Argument Generation](https://aclanthology.org/2026.emnlp-main.428/) (Maurer et al., EMNLP 2026)
ACL