@inproceedings{voronov-etal-2024-mind,
title = "Mind Your Format: Towards Consistent Evaluation of In-Context Learning Improvements",
author = "Voronov, Anton and
Wolf, Lena and
Ryabinin, Max",
editor = "Ku, Lun-Wei and
Martins, Andre and
Srikumar, Vivek",
booktitle = "Findings of the Association for Computational Linguistics ACL 2024",
month = aug,
year = "2024",
address = "Bangkok, Thailand and virtual meeting",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.findings-acl.375",
pages = "6287--6310",
abstract = "Large language models demonstrate a remarkable capability for learning to solve new tasks from a few examples.The $\textit{prompt template}$, or the way the input examples are formatted to obtain the prompt, is an important yet often overlooked aspect of in-context learning.In this work, we conduct a comprehensive study of the template format{'}s influence on the in-context learning performance.We evaluate the impact of the prompt template across 21 models (from 770M to 70B parameters) and 4 standard classification datasets. We show that a poor choice of the template can reduce the performance of the strongest models and inference methods to a random guess level.More importantly, the best templates do not transfer between different setups and even between models of the same family.Our findings show that the currently prevalent approach to evaluation, which ignores template selection, may give misleading results due to different templates in different works.As a first step towards mitigating this issue, we propose $\textit{Template Ensembles}$ that aggregate model predictions across several templates.This simple test-time augmentation boosts average performance while being robust to the choice of random set of templates.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="voronov-etal-2024-mind">
<titleInfo>
<title>Mind Your Format: Towards Consistent Evaluation of In-Context Learning Improvements</title>
</titleInfo>
<name type="personal">
<namePart type="given">Anton</namePart>
<namePart type="family">Voronov</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lena</namePart>
<namePart type="family">Wolf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Max</namePart>
<namePart type="family">Ryabinin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-08</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics ACL 2024</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lun-Wei</namePart>
<namePart type="family">Ku</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andre</namePart>
<namePart type="family">Martins</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vivek</namePart>
<namePart type="family">Srikumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Bangkok, Thailand and virtual meeting</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Large language models demonstrate a remarkable capability for learning to solve new tasks from a few examples.The prompt template, or the way the input examples are formatted to obtain the prompt, is an important yet often overlooked aspect of in-context learning.In this work, we conduct a comprehensive study of the template format’s influence on the in-context learning performance.We evaluate the impact of the prompt template across 21 models (from 770M to 70B parameters) and 4 standard classification datasets. We show that a poor choice of the template can reduce the performance of the strongest models and inference methods to a random guess level.More importantly, the best templates do not transfer between different setups and even between models of the same family.Our findings show that the currently prevalent approach to evaluation, which ignores template selection, may give misleading results due to different templates in different works.As a first step towards mitigating this issue, we propose Template Ensembles that aggregate model predictions across several templates.This simple test-time augmentation boosts average performance while being robust to the choice of random set of templates.</abstract>
<identifier type="citekey">voronov-etal-2024-mind</identifier>
<location>
<url>https://aclanthology.org/2024.findings-acl.375</url>
</location>
<part>
<date>2024-08</date>
<extent unit="page">
<start>6287</start>
<end>6310</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Mind Your Format: Towards Consistent Evaluation of In-Context Learning Improvements
%A Voronov, Anton
%A Wolf, Lena
%A Ryabinin, Max
%Y Ku, Lun-Wei
%Y Martins, Andre
%Y Srikumar, Vivek
%S Findings of the Association for Computational Linguistics ACL 2024
%D 2024
%8 August
%I Association for Computational Linguistics
%C Bangkok, Thailand and virtual meeting
%F voronov-etal-2024-mind
%X Large language models demonstrate a remarkable capability for learning to solve new tasks from a few examples.The prompt template, or the way the input examples are formatted to obtain the prompt, is an important yet often overlooked aspect of in-context learning.In this work, we conduct a comprehensive study of the template format’s influence on the in-context learning performance.We evaluate the impact of the prompt template across 21 models (from 770M to 70B parameters) and 4 standard classification datasets. We show that a poor choice of the template can reduce the performance of the strongest models and inference methods to a random guess level.More importantly, the best templates do not transfer between different setups and even between models of the same family.Our findings show that the currently prevalent approach to evaluation, which ignores template selection, may give misleading results due to different templates in different works.As a first step towards mitigating this issue, we propose Template Ensembles that aggregate model predictions across several templates.This simple test-time augmentation boosts average performance while being robust to the choice of random set of templates.
%U https://aclanthology.org/2024.findings-acl.375
%P 6287-6310
Markdown (Informal)
[Mind Your Format: Towards Consistent Evaluation of In-Context Learning Improvements](https://aclanthology.org/2024.findings-acl.375) (Voronov et al., Findings 2024)
ACL