@inproceedings{aoki-etal-2024-first,
title = "First Heuristic Then Rational: Dynamic Use of Heuristics in Language Model Reasoning",
author = "Aoki, Yoichi and
Kudo, Keito and
Kuribayashi, Tatsuki and
Sone, Shusaku and
Taniguchi, Masaya and
Sakaguchi, Keisuke and
Inui, Kentaro",
editor = "Al-Onaizan, Yaser and
Bansal, Mohit and
Chen, Yun-Nung",
booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing",
month = nov,
year = "2024",
address = "Miami, Florida, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.emnlp-main.789",
doi = "10.18653/v1/2024.emnlp-main.789",
pages = "14255--14271",
abstract = "Explicit multi-step reasoning, such as chain-of-thought, is widely adopted in the community to explore the better performance of language models (LMs). We report on the systematic strategy that LMs use in this process.Our controlled experiments reveal that LMs rely more heavily on heuristics, such as lexical overlap, in the earlier stages of reasoning when more steps are required to reach an answer. Conversely, their reliance on heuristics decreases as LMs progress closer to the final answer. This suggests that LMs track only a limited number of future steps and dynamically combine heuristic strategies with rational ones in solving tasks involving multi-step reasoning.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="aoki-etal-2024-first">
<titleInfo>
<title>First Heuristic Then Rational: Dynamic Use of Heuristics in Language Model Reasoning</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yoichi</namePart>
<namePart type="family">Aoki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keito</namePart>
<namePart type="family">Kudo</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tatsuki</namePart>
<namePart type="family">Kuribayashi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shusaku</namePart>
<namePart type="family">Sone</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Masaya</namePart>
<namePart type="family">Taniguchi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keisuke</namePart>
<namePart type="family">Sakaguchi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kentaro</namePart>
<namePart type="family">Inui</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yaser</namePart>
<namePart type="family">Al-Onaizan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohit</namePart>
<namePart type="family">Bansal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yun-Nung</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Miami, Florida, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Explicit multi-step reasoning, such as chain-of-thought, is widely adopted in the community to explore the better performance of language models (LMs). We report on the systematic strategy that LMs use in this process.Our controlled experiments reveal that LMs rely more heavily on heuristics, such as lexical overlap, in the earlier stages of reasoning when more steps are required to reach an answer. Conversely, their reliance on heuristics decreases as LMs progress closer to the final answer. This suggests that LMs track only a limited number of future steps and dynamically combine heuristic strategies with rational ones in solving tasks involving multi-step reasoning.</abstract>
<identifier type="citekey">aoki-etal-2024-first</identifier>
<identifier type="doi">10.18653/v1/2024.emnlp-main.789</identifier>
<location>
<url>https://aclanthology.org/2024.emnlp-main.789</url>
</location>
<part>
<date>2024-11</date>
<extent unit="page">
<start>14255</start>
<end>14271</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T First Heuristic Then Rational: Dynamic Use of Heuristics in Language Model Reasoning
%A Aoki, Yoichi
%A Kudo, Keito
%A Kuribayashi, Tatsuki
%A Sone, Shusaku
%A Taniguchi, Masaya
%A Sakaguchi, Keisuke
%A Inui, Kentaro
%Y Al-Onaizan, Yaser
%Y Bansal, Mohit
%Y Chen, Yun-Nung
%S Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing
%D 2024
%8 November
%I Association for Computational Linguistics
%C Miami, Florida, USA
%F aoki-etal-2024-first
%X Explicit multi-step reasoning, such as chain-of-thought, is widely adopted in the community to explore the better performance of language models (LMs). We report on the systematic strategy that LMs use in this process.Our controlled experiments reveal that LMs rely more heavily on heuristics, such as lexical overlap, in the earlier stages of reasoning when more steps are required to reach an answer. Conversely, their reliance on heuristics decreases as LMs progress closer to the final answer. This suggests that LMs track only a limited number of future steps and dynamically combine heuristic strategies with rational ones in solving tasks involving multi-step reasoning.
%R 10.18653/v1/2024.emnlp-main.789
%U https://aclanthology.org/2024.emnlp-main.789
%U https://doi.org/10.18653/v1/2024.emnlp-main.789
%P 14255-14271
Markdown (Informal)
[First Heuristic Then Rational: Dynamic Use of Heuristics in Language Model Reasoning](https://aclanthology.org/2024.emnlp-main.789) (Aoki et al., EMNLP 2024)
ACL
- Yoichi Aoki, Keito Kudo, Tatsuki Kuribayashi, Shusaku Sone, Masaya Taniguchi, Keisuke Sakaguchi, and Kentaro Inui. 2024. First Heuristic Then Rational: Dynamic Use of Heuristics in Language Model Reasoning. In Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pages 14255–14271, Miami, Florida, USA. Association for Computational Linguistics.