@inproceedings{zeng-etal-2025-enhanced,
title = "Enhanced Evaluative Language Annotation through Refined Theoretical Framework and Workflow",
author = "Zeng, Jiamei and
Wang, Haitao and
Bunt, Harry and
Cao, Xinyu and
Cardey, Sylviane and
Dong, Min and
Hao, Tianyong and
Jia, Yangli and
Lee, Kiyong and
Liao, Shengqing and
Pustejovsky, James and
Rey, Fran{\c{c}}ois Claude and
Romary, Laurent and
Zong, Jianfang and
Fang, Alex C.",
editor = "Harry, Bunt",
booktitle = "Proceedings of the 21st Joint ACL - ISO Workshop on Interoperable Semantic Annotation (ISA-21)",
month = sep,
year = "2025",
address = {D{\"u}sseldorf, Germany},
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2025.isa-1.8/",
pages = "76--84",
ISBN = "979-8-89176-319-7",
abstract = "As precursor work in preparation for an international standard \textit{ISO/PWI 24617-16 Language resource management {--} Semantic annotation {--} Part 16: Evaluative language}, we aim to test and enhance the reliability of the annotation of subjective evaluation based on Appraisal Theory. We describe a comprehensive three-phase workflow tested on COVID-19 media reports to achieve reliable agreement through progressive training and quality control. Our methodology addresses some of the key challenges through the refinement of targeted guideline refinements and the development of interactive clarification tools, alongside a custom platform that enables the pre-classification of six evaluative categories, systematic annotation review, and organized documentation. We report empirical results that demonstrate substantial improvements from the initial moderate agreement to a strong final consensus. Our research offers both theoretical refinements addressing persistent classification challenges in evaluation and practical solutions for the implementation of the annotation workflow, proposing a replicable methodology for the achievement of reliable annotation consistency in the annotation of evaluative language."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zeng-etal-2025-enhanced">
<titleInfo>
<title>Enhanced Evaluative Language Annotation through Refined Theoretical Framework and Workflow</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jiamei</namePart>
<namePart type="family">Zeng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haitao</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Harry</namePart>
<namePart type="family">Bunt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sylviane</namePart>
<namePart type="family">Cardey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Min</namePart>
<namePart type="family">Dong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianyong</namePart>
<namePart type="family">Hao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yangli</namePart>
<namePart type="family">Jia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kiyong</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shengqing</namePart>
<namePart type="family">Liao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">James</namePart>
<namePart type="family">Pustejovsky</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">François</namePart>
<namePart type="given">Claude</namePart>
<namePart type="family">Rey</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Laurent</namePart>
<namePart type="family">Romary</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jianfang</namePart>
<namePart type="family">Zong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="given">C</namePart>
<namePart type="family">Fang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2025-09</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 21st Joint ACL - ISO Workshop on Interoperable Semantic Annotation (ISA-21)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Bunt</namePart>
<namePart type="family">Harry</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Düsseldorf, Germany</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-319-7</identifier>
</relatedItem>
<abstract>As precursor work in preparation for an international standard ISO/PWI 24617-16 Language resource management – Semantic annotation – Part 16: Evaluative language, we aim to test and enhance the reliability of the annotation of subjective evaluation based on Appraisal Theory. We describe a comprehensive three-phase workflow tested on COVID-19 media reports to achieve reliable agreement through progressive training and quality control. Our methodology addresses some of the key challenges through the refinement of targeted guideline refinements and the development of interactive clarification tools, alongside a custom platform that enables the pre-classification of six evaluative categories, systematic annotation review, and organized documentation. We report empirical results that demonstrate substantial improvements from the initial moderate agreement to a strong final consensus. Our research offers both theoretical refinements addressing persistent classification challenges in evaluation and practical solutions for the implementation of the annotation workflow, proposing a replicable methodology for the achievement of reliable annotation consistency in the annotation of evaluative language.</abstract>
<identifier type="citekey">zeng-etal-2025-enhanced</identifier>
<location>
<url>https://aclanthology.org/2025.isa-1.8/</url>
</location>
<part>
<date>2025-09</date>
<extent unit="page">
<start>76</start>
<end>84</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Enhanced Evaluative Language Annotation through Refined Theoretical Framework and Workflow
%A Zeng, Jiamei
%A Wang, Haitao
%A Bunt, Harry
%A Cao, Xinyu
%A Cardey, Sylviane
%A Dong, Min
%A Hao, Tianyong
%A Jia, Yangli
%A Lee, Kiyong
%A Liao, Shengqing
%A Pustejovsky, James
%A Rey, François Claude
%A Romary, Laurent
%A Zong, Jianfang
%A Fang, Alex C.
%Y Harry, Bunt
%S Proceedings of the 21st Joint ACL - ISO Workshop on Interoperable Semantic Annotation (ISA-21)
%D 2025
%8 September
%I Association for Computational Linguistics
%C Düsseldorf, Germany
%@ 979-8-89176-319-7
%F zeng-etal-2025-enhanced
%X As precursor work in preparation for an international standard ISO/PWI 24617-16 Language resource management – Semantic annotation – Part 16: Evaluative language, we aim to test and enhance the reliability of the annotation of subjective evaluation based on Appraisal Theory. We describe a comprehensive three-phase workflow tested on COVID-19 media reports to achieve reliable agreement through progressive training and quality control. Our methodology addresses some of the key challenges through the refinement of targeted guideline refinements and the development of interactive clarification tools, alongside a custom platform that enables the pre-classification of six evaluative categories, systematic annotation review, and organized documentation. We report empirical results that demonstrate substantial improvements from the initial moderate agreement to a strong final consensus. Our research offers both theoretical refinements addressing persistent classification challenges in evaluation and practical solutions for the implementation of the annotation workflow, proposing a replicable methodology for the achievement of reliable annotation consistency in the annotation of evaluative language.
%U https://aclanthology.org/2025.isa-1.8/
%P 76-84
Markdown (Informal)
[Enhanced Evaluative Language Annotation through Refined Theoretical Framework and Workflow](https://aclanthology.org/2025.isa-1.8/) (Zeng et al., ISA 2025)
ACL
- Jiamei Zeng, Haitao Wang, Harry Bunt, Xinyu Cao, Sylviane Cardey, Min Dong, Tianyong Hao, Yangli Jia, Kiyong Lee, Shengqing Liao, James Pustejovsky, François Claude Rey, Laurent Romary, Jianfang Zong, and Alex C. Fang. 2025. Enhanced Evaluative Language Annotation through Refined Theoretical Framework and Workflow. In Proceedings of the 21st Joint ACL - ISO Workshop on Interoperable Semantic Annotation (ISA-21), pages 76–84, Düsseldorf, Germany. Association for Computational Linguistics.