@inproceedings{meng-etal-2026-releaseeval,
title = "{R}elease{E}val: A Benchmark for Evaluating Language Models in Automated Release Note Generation",
author = "Meng, Qianru and
Ren, Zhaochun and
Visser, Joost",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.128/",
pages = "2514--2526",
ISBN = "979-8-89176-444-6",
abstract = "Automated release note generation aims to reduce the cost of documenting frequent software updates. Existing benchmarks mainly rely on commit messages, limiting evaluation of how models use broader software-change evidence. We introduce ReleaseEval, a reproducible and openly licensed multi-granularity benchmark with 94,987 release notes from 3,369 repositories across six programming languages. Each release is aligned with commit messages, commit tree structures, and raw code diffs, supporting three task settings: commit2sum, tree2sum, and diff2sum. Automatic and human evaluations show that models benefit from lightweight commit-tree structure, while raw diffs remain challenging due to length and noise. These findings position ReleaseEval as a multi-granularity benchmark for studying release note generation. We release the dataset and code at https://github.com/Changanyuexia/rn{\_}benchmark/"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="meng-etal-2026-releaseeval">
<titleInfo>
<title>ReleaseEval: A Benchmark for Evaluating Language Models in Automated Release Note Generation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Qianru</namePart>
<namePart type="family">Meng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhaochun</namePart>
<namePart type="family">Ren</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Joost</namePart>
<namePart type="family">Visser</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Automated release note generation aims to reduce the cost of documenting frequent software updates. Existing benchmarks mainly rely on commit messages, limiting evaluation of how models use broader software-change evidence. We introduce ReleaseEval, a reproducible and openly licensed multi-granularity benchmark with 94,987 release notes from 3,369 repositories across six programming languages. Each release is aligned with commit messages, commit tree structures, and raw code diffs, supporting three task settings: commit2sum, tree2sum, and diff2sum. Automatic and human evaluations show that models benefit from lightweight commit-tree structure, while raw diffs remain challenging due to length and noise. These findings position ReleaseEval as a multi-granularity benchmark for studying release note generation. We release the dataset and code at https://github.com/Changanyuexia/rn_benchmark/</abstract>
<identifier type="citekey">meng-etal-2026-releaseeval</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.128/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>2514</start>
<end>2526</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T ReleaseEval: A Benchmark for Evaluating Language Models in Automated Release Note Generation
%A Meng, Qianru
%A Ren, Zhaochun
%A Visser, Joost
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F meng-etal-2026-releaseeval
%X Automated release note generation aims to reduce the cost of documenting frequent software updates. Existing benchmarks mainly rely on commit messages, limiting evaluation of how models use broader software-change evidence. We introduce ReleaseEval, a reproducible and openly licensed multi-granularity benchmark with 94,987 release notes from 3,369 repositories across six programming languages. Each release is aligned with commit messages, commit tree structures, and raw code diffs, supporting three task settings: commit2sum, tree2sum, and diff2sum. Automatic and human evaluations show that models benefit from lightweight commit-tree structure, while raw diffs remain challenging due to length and noise. These findings position ReleaseEval as a multi-granularity benchmark for studying release note generation. We release the dataset and code at https://github.com/Changanyuexia/rn_benchmark/
%U https://aclanthology.org/2026.findings-aacl.128/
%P 2514-2526
Markdown (Informal)
[ReleaseEval: A Benchmark for Evaluating Language Models in Automated Release Note Generation](https://aclanthology.org/2026.findings-aacl.128/) (Meng et al., Findings 2026)
ACL