@inproceedings{saha-etal-2026-assessing,
title = "Assessing Large Language Models as Automated Writing Evaluation and Feedback Tools: A Human-{AI} Agreement Study on {L}2 {E}nglish Writing",
author = "Saha, Kowshik and
Uddin, Moyen and
Turad, Md Tanvirul Islam and
Rahman, Mizanur",
editor = "Zhang, Chen and
Allaway, Emily and
M{'}hamdi, Meryem and
Limkonchotiwat, Peerat and
Bai, Richard He and
T.Y.S.S., Santosh and
Thapa, Surendrabikram and
Li, Yinqiao",
booktitle = "Proceedings of the 10th Widening {NLP} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.winlp-main.10/",
pages = "96--104",
ISBN = "979-8-89176-472-9",
abstract = "Large language models (LLMs) are often suggested as tools for grading student writing and giving feedback, but it is unclear how far we can trust them on the separate skills that make up good writing. We compare two LLM graders (Gemini and Llama) with eight professional English teachers on 20 paragraphs written in second-language English by Bengali-speaking undergraduates in Bangladesh, each graded by two teachers, scoring content, organization, vocabulary, and grammar separately as well as the total. We then run a second stage in which teachers read each model{'}s written explanation, judge whether they agree with it, and rate the feedback. The LLMs match the teachers on the total score, content, and organization, but are unreliable on vocabulary and grammar, where the two models do not even agree with each other. We also show that one low agreement score is misleading: it comes from graders giving very similar marks (score clustering), not from real disagreement. In the second stage, teachers found the AI feedback safe and useful most of the time ($\geq$ 77.5{\%}), with very few made-up errors (2.5{\%}), and preferred Gemini{'}s stricter grammar score to their own 20{\%} of the time. LLMs can help grade writing, but a teacher should audit their output, with extra care on vocabulary and grammar."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="saha-etal-2026-assessing">
<titleInfo>
<title>Assessing Large Language Models as Automated Writing Evaluation and Feedback Tools: A Human-AI Agreement Study on L2 English Writing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kowshik</namePart>
<namePart type="family">Saha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Moyen</namePart>
<namePart type="family">Uddin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Tanvirul</namePart>
<namePart type="given">Islam</namePart>
<namePart type="family">Turad</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mizanur</namePart>
<namePart type="family">Rahman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 10th Widening NLP Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Chen</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Emily</namePart>
<namePart type="family">Allaway</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Meryem</namePart>
<namePart type="family">M’hamdi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Peerat</namePart>
<namePart type="family">Limkonchotiwat</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Richard</namePart>
<namePart type="given">He</namePart>
<namePart type="family">Bai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Santosh</namePart>
<namePart type="family">T.Y.S.S.</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Surendrabikram</namePart>
<namePart type="family">Thapa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yinqiao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-472-9</identifier>
</relatedItem>
<abstract>Large language models (LLMs) are often suggested as tools for grading student writing and giving feedback, but it is unclear how far we can trust them on the separate skills that make up good writing. We compare two LLM graders (Gemini and Llama) with eight professional English teachers on 20 paragraphs written in second-language English by Bengali-speaking undergraduates in Bangladesh, each graded by two teachers, scoring content, organization, vocabulary, and grammar separately as well as the total. We then run a second stage in which teachers read each model’s written explanation, judge whether they agree with it, and rate the feedback. The LLMs match the teachers on the total score, content, and organization, but are unreliable on vocabulary and grammar, where the two models do not even agree with each other. We also show that one low agreement score is misleading: it comes from graders giving very similar marks (score clustering), not from real disagreement. In the second stage, teachers found the AI feedback safe and useful most of the time (\geq 77.5%), with very few made-up errors (2.5%), and preferred Gemini’s stricter grammar score to their own 20% of the time. LLMs can help grade writing, but a teacher should audit their output, with extra care on vocabulary and grammar.</abstract>
<identifier type="citekey">saha-etal-2026-assessing</identifier>
<location>
<url>https://aclanthology.org/2026.winlp-main.10/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>96</start>
<end>104</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Assessing Large Language Models as Automated Writing Evaluation and Feedback Tools: A Human-AI Agreement Study on L2 English Writing
%A Saha, Kowshik
%A Uddin, Moyen
%A Turad, Md Tanvirul Islam
%A Rahman, Mizanur
%Y Zhang, Chen
%Y Allaway, Emily
%Y M’hamdi, Meryem
%Y Limkonchotiwat, Peerat
%Y Bai, Richard He
%Y T.Y.S.S., Santosh
%Y Thapa, Surendrabikram
%Y Li, Yinqiao
%S Proceedings of the 10th Widening NLP Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-472-9
%F saha-etal-2026-assessing
%X Large language models (LLMs) are often suggested as tools for grading student writing and giving feedback, but it is unclear how far we can trust them on the separate skills that make up good writing. We compare two LLM graders (Gemini and Llama) with eight professional English teachers on 20 paragraphs written in second-language English by Bengali-speaking undergraduates in Bangladesh, each graded by two teachers, scoring content, organization, vocabulary, and grammar separately as well as the total. We then run a second stage in which teachers read each model’s written explanation, judge whether they agree with it, and rate the feedback. The LLMs match the teachers on the total score, content, and organization, but are unreliable on vocabulary and grammar, where the two models do not even agree with each other. We also show that one low agreement score is misleading: it comes from graders giving very similar marks (score clustering), not from real disagreement. In the second stage, teachers found the AI feedback safe and useful most of the time (\geq 77.5%), with very few made-up errors (2.5%), and preferred Gemini’s stricter grammar score to their own 20% of the time. LLMs can help grade writing, but a teacher should audit their output, with extra care on vocabulary and grammar.
%U https://aclanthology.org/2026.winlp-main.10/
%P 96-104
Markdown (Informal)
[Assessing Large Language Models as Automated Writing Evaluation and Feedback Tools: A Human-AI Agreement Study on L2 English Writing](https://aclanthology.org/2026.winlp-main.10/) (Saha et al., WiNLP 2026)
ACL