@inproceedings{ying-etal-2026-openskilleval,
title = "{O}pen{S}kill{E}val: Automatically Auditing the Open Skill Ecosystem for {LLM} Agents",
author = "Ying, Jiahao and
Ai, Boxian and
Tang, Wei and
Liu, Siyuan and
Cao, Yixin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1554/",
pages = "33083--33111",
ISBN = "979-8-89176-462-0",
abstract = "Skills are increasingly used to improve LLM agents on real-world tasks, yet it remains unclear how agents use skills, how skill quality should be evaluated, and how users should choose skills under cost-performance trade-offs. We present OpenSkillEval, an automatic framework for evaluating both skill-augmented agents and skills. OpenSkillEval dynamically constructs test cases from evolving real-world artifacts across five applications: presentation generation, front-end web design, poster generation, data visualization, and report generation. It also curates community skills for controlled comparison under unified settings. Using over 600 generated tasks and 30 open-source skills, we evaluate state-of-the-art models and agent frameworks. Results show that skill availability does not guarantee effective use, skill benefits depend strongly on the model and framework, and many popular skills fail to consistently outperform skill-free agents. These findings highlight the need for dynamic, task-grounded evaluation and offer practical guidance for skill design, selection, and deployment."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ying-etal-2026-openskilleval">
<titleInfo>
<title>OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiahao</namePart>
<namePart type="family">Ying</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Boxian</namePart>
<namePart type="family">Ai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyuan</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yixin</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Skills are increasingly used to improve LLM agents on real-world tasks, yet it remains unclear how agents use skills, how skill quality should be evaluated, and how users should choose skills under cost-performance trade-offs. We present OpenSkillEval, an automatic framework for evaluating both skill-augmented agents and skills. OpenSkillEval dynamically constructs test cases from evolving real-world artifacts across five applications: presentation generation, front-end web design, poster generation, data visualization, and report generation. It also curates community skills for controlled comparison under unified settings. Using over 600 generated tasks and 30 open-source skills, we evaluate state-of-the-art models and agent frameworks. Results show that skill availability does not guarantee effective use, skill benefits depend strongly on the model and framework, and many popular skills fail to consistently outperform skill-free agents. These findings highlight the need for dynamic, task-grounded evaluation and offer practical guidance for skill design, selection, and deployment.</abstract>
<identifier type="citekey">ying-etal-2026-openskilleval</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1554/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>33083</start>
<end>33111</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents
%A Ying, Jiahao
%A Ai, Boxian
%A Tang, Wei
%A Liu, Siyuan
%A Cao, Yixin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F ying-etal-2026-openskilleval
%X Skills are increasingly used to improve LLM agents on real-world tasks, yet it remains unclear how agents use skills, how skill quality should be evaluated, and how users should choose skills under cost-performance trade-offs. We present OpenSkillEval, an automatic framework for evaluating both skill-augmented agents and skills. OpenSkillEval dynamically constructs test cases from evolving real-world artifacts across five applications: presentation generation, front-end web design, poster generation, data visualization, and report generation. It also curates community skills for controlled comparison under unified settings. Using over 600 generated tasks and 30 open-source skills, we evaluate state-of-the-art models and agent frameworks. Results show that skill availability does not guarantee effective use, skill benefits depend strongly on the model and framework, and many popular skills fail to consistently outperform skill-free agents. These findings highlight the need for dynamic, task-grounded evaluation and offer practical guidance for skill design, selection, and deployment.
%U https://aclanthology.org/2026.findings-emnlp.1554/
%P 33083-33111
Markdown (Informal)
[OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents](https://aclanthology.org/2026.findings-emnlp.1554/) (Ying et al., Findings 2026)
ACL