@inproceedings{tsuji-etal-2026-many,
title = "Are Many Tasks Necessary for General {LLM} Evaluation? A Matrix Completion View of Benchmark Redundancy",
author = "Tsuji, Kohei and
Sakai, Yusuke and
Kamigaito, Hidetaka and
Watanabe, Taro",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.55/",
pages = "1172--1187",
ISBN = "979-8-89176-439-2",
abstract = "Large language model (LLM) evaluation usually averages scores across many benchmark tasks, implicitly assuming broad task coverage is necessary for reliable general evaluation. This practice also implicitly treats all benchmark tasks as equally important for general evaluation. We revisit this assumption by treating benchmark results as a model-task score matrix under the performance prediction task. We find that a broad range of benchmarks are redundant with scores for 100 LLMs on 13 aggregate tasks and 523 subtasks. Specifically, we recovered model rankings with high rank correlation with the ALS matrix completion method after masking out several task scores. This means that no tasks dominated the recovery process, and both low-scoring and high-scoring tasks shared the same trends. Further analyses show that multiple-choice scores allow us to recover model rankings on free-form generation tasks. For general model ranking, free-form generation evaluation is not necessarily more informative than multiple-choice evaluation. These findings suggest that evaluating general-purpose LLMs can be accomplished with fewer tasks than the current common practice. Our analysis also estimates the number of models required to capture the overall trends in the benchmarks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tsuji-etal-2026-many">
<titleInfo>
<title>Are Many Tasks Necessary for General LLM Evaluation? A Matrix Completion View of Benchmark Redundancy</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kohei</namePart>
<namePart type="family">Tsuji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yusuke</namePart>
<namePart type="family">Sakai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hidetaka</namePart>
<namePart type="family">Kamigaito</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Taro</namePart>
<namePart type="family">Watanabe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>Large language model (LLM) evaluation usually averages scores across many benchmark tasks, implicitly assuming broad task coverage is necessary for reliable general evaluation. This practice also implicitly treats all benchmark tasks as equally important for general evaluation. We revisit this assumption by treating benchmark results as a model-task score matrix under the performance prediction task. We find that a broad range of benchmarks are redundant with scores for 100 LLMs on 13 aggregate tasks and 523 subtasks. Specifically, we recovered model rankings with high rank correlation with the ALS matrix completion method after masking out several task scores. This means that no tasks dominated the recovery process, and both low-scoring and high-scoring tasks shared the same trends. Further analyses show that multiple-choice scores allow us to recover model rankings on free-form generation tasks. For general model ranking, free-form generation evaluation is not necessarily more informative than multiple-choice evaluation. These findings suggest that evaluating general-purpose LLMs can be accomplished with fewer tasks than the current common practice. Our analysis also estimates the number of models required to capture the overall trends in the benchmarks.</abstract>
<identifier type="citekey">tsuji-etal-2026-many</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.55/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>1172</start>
<end>1187</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Are Many Tasks Necessary for General LLM Evaluation? A Matrix Completion View of Benchmark Redundancy
%A Tsuji, Kohei
%A Sakai, Yusuke
%A Kamigaito, Hidetaka
%A Watanabe, Taro
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F tsuji-etal-2026-many
%X Large language model (LLM) evaluation usually averages scores across many benchmark tasks, implicitly assuming broad task coverage is necessary for reliable general evaluation. This practice also implicitly treats all benchmark tasks as equally important for general evaluation. We revisit this assumption by treating benchmark results as a model-task score matrix under the performance prediction task. We find that a broad range of benchmarks are redundant with scores for 100 LLMs on 13 aggregate tasks and 523 subtasks. Specifically, we recovered model rankings with high rank correlation with the ALS matrix completion method after masking out several task scores. This means that no tasks dominated the recovery process, and both low-scoring and high-scoring tasks shared the same trends. Further analyses show that multiple-choice scores allow us to recover model rankings on free-form generation tasks. For general model ranking, free-form generation evaluation is not necessarily more informative than multiple-choice evaluation. These findings suggest that evaluating general-purpose LLMs can be accomplished with fewer tasks than the current common practice. Our analysis also estimates the number of models required to capture the overall trends in the benchmarks.
%U https://aclanthology.org/2026.aacl-long.55/
%P 1172-1187
Markdown (Informal)
[Are Many Tasks Necessary for General LLM Evaluation? A Matrix Completion View of Benchmark Redundancy](https://aclanthology.org/2026.aacl-long.55/) (Tsuji et al., AACL-IJCNLP 2026)
ACL