@inproceedings{albatarni-etal-2026-rethinking,
title = "Rethinking Evaluation Practices in Automated Essay Scoring: A Quantitative Analysis over the Past Decade",
author = "Albatarni, Salam and
Bashendy, May and
Eltanbouly, Sohaila and
Sayed, Marwan and
Elsayed, Tamer",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-main.1963/",
pages = "44705--44721",
ISBN = "979-8-89176-459-0",
abstract = "Automated Essay Scoring (AES) aims to automatically assign quality scores to written essays and has been studied for several decades. Despite substantial methodological progress, little attention has been given to how AES systems are evaluated. In this position paper, we argue that several common AES evaluation practices are misaligned with broader machine learning evaluation standards. To examine current practices, we systematically analyzed 101 AES papers published in leading venues over the past decade, focusing on experimental setup, reproducibility, model development, and performance reporting and analysis. Consequently, we identify recurring shortcomings and highlight ways for more rigorous and transparent evaluation practices."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="albatarni-etal-2026-rethinking">
<titleInfo>
<title>Rethinking Evaluation Practices in Automated Essay Scoring: A Quantitative Analysis over the Past Decade</title>
</titleInfo>
<name type="personal">
<namePart type="given">Salam</namePart>
<namePart type="family">Albatarni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">May</namePart>
<namePart type="family">Bashendy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sohaila</namePart>
<namePart type="family">Eltanbouly</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Marwan</namePart>
<namePart type="family">Sayed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamer</namePart>
<namePart type="family">Elsayed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-459-0</identifier>
</relatedItem>
<abstract>Automated Essay Scoring (AES) aims to automatically assign quality scores to written essays and has been studied for several decades. Despite substantial methodological progress, little attention has been given to how AES systems are evaluated. In this position paper, we argue that several common AES evaluation practices are misaligned with broader machine learning evaluation standards. To examine current practices, we systematically analyzed 101 AES papers published in leading venues over the past decade, focusing on experimental setup, reproducibility, model development, and performance reporting and analysis. Consequently, we identify recurring shortcomings and highlight ways for more rigorous and transparent evaluation practices.</abstract>
<identifier type="citekey">albatarni-etal-2026-rethinking</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-main.1963/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>44705</start>
<end>44721</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Rethinking Evaluation Practices in Automated Essay Scoring: A Quantitative Analysis over the Past Decade
%A Albatarni, Salam
%A Bashendy, May
%A Eltanbouly, Sohaila
%A Sayed, Marwan
%A Elsayed, Tamer
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-459-0
%F albatarni-etal-2026-rethinking
%X Automated Essay Scoring (AES) aims to automatically assign quality scores to written essays and has been studied for several decades. Despite substantial methodological progress, little attention has been given to how AES systems are evaluated. In this position paper, we argue that several common AES evaluation practices are misaligned with broader machine learning evaluation standards. To examine current practices, we systematically analyzed 101 AES papers published in leading venues over the past decade, focusing on experimental setup, reproducibility, model development, and performance reporting and analysis. Consequently, we identify recurring shortcomings and highlight ways for more rigorous and transparent evaluation practices.
%U https://aclanthology.org/2026.emnlp-main.1963/
%P 44705-44721
Markdown (Informal)
[Rethinking Evaluation Practices in Automated Essay Scoring: A Quantitative Analysis over the Past Decade](https://aclanthology.org/2026.emnlp-main.1963/) (Albatarni et al., EMNLP 2026)
ACL