@inproceedings{georgousis-etal-2026-evaluating,
title = "Evaluating Counterfactual Strategic Reasoning in Large Language Models",
author = "Georgousis, Dimitrios and
Lymperaiou, Maria and
Dimitriou, Angeliki and
Filandrianos, Giorgos and
Stamou, Giorgos",
editor = "Mille, Simon and
Gehrmann, Sebastian and
Schmidtov{\'a}, Patr{\'i}cia and
Du{\v{s}}ek, Ond{\v{r}}ej and
Fadaee, Marzieh and
Lo, Kyle and
Santus, Enrico and
Stanovsky, Gabriel",
booktitle = "Proceedings of the Fifth Workshop on Generation, Evaluation and Metrics ({GEM})",
month = jul,
year = "2026",
address = "San Diego, California, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.gem-main.31/",
pages = "309--354",
ISBN = "979-8-89176-423-1",
abstract = "We evaluate whether LLMs adapt their strategic behavior when familiar games are counterfactually modified. We introduce a repeated-game evaluation framework covering Prisoner{'}s Dilemma and Rock{--}Paper{--}Scissors under default, label-perturbed, payoff-perturbed, and joint counterfactual variants. This design separates surface robustness to renamed actions from deeper sensitivity to changed incentives. Across multiple frontier LLMs, we find that label perturbations usually cause moderate degradation, whereas payoff perturbations expose stronger failures: LLMs often preserve canonical strategies even when the equilibrium structure changes. In RPS, several LLMs remain close to uniform play despite a payoff-counterfactual equilibrium requiring a biased mixed strategy. Behavioral and efficiency metrics further show that stronger or reasoning-enabled LLMs are not uniformly more strategic: some deliberate more without adapting faster. Overall, counterfactual repeated games provide a compact diagnostic for distinguishing robust incentive-sensitive behavior from brittle template-based strategic execution."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="georgousis-etal-2026-evaluating">
<titleInfo>
<title>Evaluating Counterfactual Strategic Reasoning in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dimitrios</namePart>
<namePart type="family">Georgousis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Maria</namePart>
<namePart type="family">Lymperaiou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Angeliki</namePart>
<namePart type="family">Dimitriou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giorgos</namePart>
<namePart type="family">Filandrianos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Giorgos</namePart>
<namePart type="family">Stamou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifth Workshop on Generation, Evaluation and Metrics (GEM)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Mille</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sebastian</namePart>
<namePart type="family">Gehrmann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Patrícia</namePart>
<namePart type="family">Schmidtová</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ondřej</namePart>
<namePart type="family">Dušek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marzieh</namePart>
<namePart type="family">Fadaee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kyle</namePart>
<namePart type="family">Lo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Enrico</namePart>
<namePart type="family">Santus</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriel</namePart>
<namePart type="family">Stanovsky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, California, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-423-1</identifier>
</relatedItem>
<abstract>We evaluate whether LLMs adapt their strategic behavior when familiar games are counterfactually modified. We introduce a repeated-game evaluation framework covering Prisoner’s Dilemma and Rock–Paper–Scissors under default, label-perturbed, payoff-perturbed, and joint counterfactual variants. This design separates surface robustness to renamed actions from deeper sensitivity to changed incentives. Across multiple frontier LLMs, we find that label perturbations usually cause moderate degradation, whereas payoff perturbations expose stronger failures: LLMs often preserve canonical strategies even when the equilibrium structure changes. In RPS, several LLMs remain close to uniform play despite a payoff-counterfactual equilibrium requiring a biased mixed strategy. Behavioral and efficiency metrics further show that stronger or reasoning-enabled LLMs are not uniformly more strategic: some deliberate more without adapting faster. Overall, counterfactual repeated games provide a compact diagnostic for distinguishing robust incentive-sensitive behavior from brittle template-based strategic execution.</abstract>
<identifier type="citekey">georgousis-etal-2026-evaluating</identifier>
<location>
<url>https://aclanthology.org/2026.gem-main.31/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>309</start>
<end>354</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Counterfactual Strategic Reasoning in Large Language Models
%A Georgousis, Dimitrios
%A Lymperaiou, Maria
%A Dimitriou, Angeliki
%A Filandrianos, Giorgos
%A Stamou, Giorgos
%Y Mille, Simon
%Y Gehrmann, Sebastian
%Y Schmidtová, Patrícia
%Y Dušek, Ondřej
%Y Fadaee, Marzieh
%Y Lo, Kyle
%Y Santus, Enrico
%Y Stanovsky, Gabriel
%S Proceedings of the Fifth Workshop on Generation, Evaluation and Metrics (GEM)
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, California, USA
%@ 979-8-89176-423-1
%F georgousis-etal-2026-evaluating
%X We evaluate whether LLMs adapt their strategic behavior when familiar games are counterfactually modified. We introduce a repeated-game evaluation framework covering Prisoner’s Dilemma and Rock–Paper–Scissors under default, label-perturbed, payoff-perturbed, and joint counterfactual variants. This design separates surface robustness to renamed actions from deeper sensitivity to changed incentives. Across multiple frontier LLMs, we find that label perturbations usually cause moderate degradation, whereas payoff perturbations expose stronger failures: LLMs often preserve canonical strategies even when the equilibrium structure changes. In RPS, several LLMs remain close to uniform play despite a payoff-counterfactual equilibrium requiring a biased mixed strategy. Behavioral and efficiency metrics further show that stronger or reasoning-enabled LLMs are not uniformly more strategic: some deliberate more without adapting faster. Overall, counterfactual repeated games provide a compact diagnostic for distinguishing robust incentive-sensitive behavior from brittle template-based strategic execution.
%U https://aclanthology.org/2026.gem-main.31/
%P 309-354
Markdown (Informal)
[Evaluating Counterfactual Strategic Reasoning in Large Language Models](https://aclanthology.org/2026.gem-main.31/) (Georgousis et al., GEM 2026)
ACL