@inproceedings{misra-etal-2021-accountable,
title = "Accountable Error Characterization",
author = "Misra, Amita and
Liu, Zhe and
Mahmud, Jalal",
editor = "Pruksachatkun, Yada and
Ramakrishna, Anil and
Chang, Kai-Wei and
Krishna, Satyapriya and
Dhamala, Jwala and
Guha, Tanaya and
Ren, Xiang",
booktitle = "Proceedings of the First Workshop on Trustworthy Natural Language Processing",
month = jun,
year = "2021",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.trustnlp-1.4",
doi = "10.18653/v1/2021.trustnlp-1.4",
pages = "28--33",
abstract = "Customers of machine learning systems demand accountability from the companies employing these algorithms for various prediction tasks. Accountability requires understanding of system limit and condition of erroneous predictions, as customers are often interested in understanding the incorrect predictions, and model developers are absorbed in finding methods that can be used to get incremental improvements to an existing system. Therefore, we propose an accountable error characterization method, AEC, to understand when and where errors occur within the existing black-box models. AEC, as constructed with human-understandable linguistic features, allows the model developers to automatically identify the main sources of errors for a given classification system. It can also be used to sample for the set of most informative input points for a next round of training. We perform error detection for a sentiment analysis task using AEC as a case study. Our results on the sample sentiment task show that AEC is able to characterize erroneous predictions into human understandable categories and also achieves promising results on selecting erroneous samples when compared with the uncertainty-based sampling.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="misra-etal-2021-accountable">
<titleInfo>
<title>Accountable Error Characterization</title>
</titleInfo>
<name type="personal">
<namePart type="given">Amita</namePart>
<namePart type="family">Misra</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhe</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jalal</namePart>
<namePart type="family">Mahmud</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2021-06</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on Trustworthy Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yada</namePart>
<namePart type="family">Pruksachatkun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anil</namePart>
<namePart type="family">Ramakrishna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai-Wei</namePart>
<namePart type="family">Chang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Satyapriya</namePart>
<namePart type="family">Krishna</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jwala</namePart>
<namePart type="family">Dhamala</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tanaya</namePart>
<namePart type="family">Guha</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xiang</namePart>
<namePart type="family">Ren</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Customers of machine learning systems demand accountability from the companies employing these algorithms for various prediction tasks. Accountability requires understanding of system limit and condition of erroneous predictions, as customers are often interested in understanding the incorrect predictions, and model developers are absorbed in finding methods that can be used to get incremental improvements to an existing system. Therefore, we propose an accountable error characterization method, AEC, to understand when and where errors occur within the existing black-box models. AEC, as constructed with human-understandable linguistic features, allows the model developers to automatically identify the main sources of errors for a given classification system. It can also be used to sample for the set of most informative input points for a next round of training. We perform error detection for a sentiment analysis task using AEC as a case study. Our results on the sample sentiment task show that AEC is able to characterize erroneous predictions into human understandable categories and also achieves promising results on selecting erroneous samples when compared with the uncertainty-based sampling.</abstract>
<identifier type="citekey">misra-etal-2021-accountable</identifier>
<identifier type="doi">10.18653/v1/2021.trustnlp-1.4</identifier>
<location>
<url>https://aclanthology.org/2021.trustnlp-1.4</url>
</location>
<part>
<date>2021-06</date>
<extent unit="page">
<start>28</start>
<end>33</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Accountable Error Characterization
%A Misra, Amita
%A Liu, Zhe
%A Mahmud, Jalal
%Y Pruksachatkun, Yada
%Y Ramakrishna, Anil
%Y Chang, Kai-Wei
%Y Krishna, Satyapriya
%Y Dhamala, Jwala
%Y Guha, Tanaya
%Y Ren, Xiang
%S Proceedings of the First Workshop on Trustworthy Natural Language Processing
%D 2021
%8 June
%I Association for Computational Linguistics
%C Online
%F misra-etal-2021-accountable
%X Customers of machine learning systems demand accountability from the companies employing these algorithms for various prediction tasks. Accountability requires understanding of system limit and condition of erroneous predictions, as customers are often interested in understanding the incorrect predictions, and model developers are absorbed in finding methods that can be used to get incremental improvements to an existing system. Therefore, we propose an accountable error characterization method, AEC, to understand when and where errors occur within the existing black-box models. AEC, as constructed with human-understandable linguistic features, allows the model developers to automatically identify the main sources of errors for a given classification system. It can also be used to sample for the set of most informative input points for a next round of training. We perform error detection for a sentiment analysis task using AEC as a case study. Our results on the sample sentiment task show that AEC is able to characterize erroneous predictions into human understandable categories and also achieves promising results on selecting erroneous samples when compared with the uncertainty-based sampling.
%R 10.18653/v1/2021.trustnlp-1.4
%U https://aclanthology.org/2021.trustnlp-1.4
%U https://doi.org/10.18653/v1/2021.trustnlp-1.4
%P 28-33
Markdown (Informal)
[Accountable Error Characterization](https://aclanthology.org/2021.trustnlp-1.4) (Misra et al., TrustNLP 2021)
ACL
- Amita Misra, Zhe Liu, and Jalal Mahmud. 2021. Accountable Error Characterization. In Proceedings of the First Workshop on Trustworthy Natural Language Processing, pages 28–33, Online. Association for Computational Linguistics.