@inproceedings{hou-etal-2024-course-shared,
title = "A Course Shared Task on Evaluating {LLM} Output for Clinical Questions",
author = "Hou, Yufang and
Tran, Thy and
Vu, Doan and
Cao, Yiwen and
Li, Kai and
Rohde, Lukas and
Gurevych, Iryna",
editor = {Al-azzawi, Sana and
Biester, Laura and
Kov{\'a}cs, Gy{\"o}rgy and
Marasovi{\'c}, Ana and
Mathur, Leena and
Mieskes, Margot and
Weissweiler, Leonie},
booktitle = "Proceedings of the Sixth Workshop on Teaching NLP",
month = aug,
year = "2024",
address = "Bangkok, Thailand",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.teachingnlp-1.11",
pages = "77--80",
abstract = "This paper presents a shared task that we organized at the Foundations of Language Technology (FoLT) course in 2023/2024 at the Technical University of Darmstadt, which focuses on evaluating the output of Large Language Models (LLMs) in generating harmful answers to health-related clinical questions. We describe the task design considerations and report the feedback we received from the students. We expect the task and the findings reported in this paper to be relevant for instructors teaching natural language processing (NLP).",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hou-etal-2024-course-shared">
<titleInfo>
<title>A Course Shared Task on Evaluating LLM Output for Clinical Questions</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yufang</namePart>
<namePart type="family">Hou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thy</namePart>
<namePart type="family">Tran</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Doan</namePart>
<namePart type="family">Vu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yiwen</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lukas</namePart>
<namePart type="family">Rohde</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Iryna</namePart>
<namePart type="family">Gurevych</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-08</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Sixth Workshop on Teaching NLP</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sana</namePart>
<namePart type="family">Al-azzawi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Laura</namePart>
<namePart type="family">Biester</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">György</namePart>
<namePart type="family">Kovács</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ana</namePart>
<namePart type="family">Marasović</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leena</namePart>
<namePart type="family">Mathur</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Margot</namePart>
<namePart type="family">Mieskes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Leonie</namePart>
<namePart type="family">Weissweiler</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Bangkok, Thailand</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This paper presents a shared task that we organized at the Foundations of Language Technology (FoLT) course in 2023/2024 at the Technical University of Darmstadt, which focuses on evaluating the output of Large Language Models (LLMs) in generating harmful answers to health-related clinical questions. We describe the task design considerations and report the feedback we received from the students. We expect the task and the findings reported in this paper to be relevant for instructors teaching natural language processing (NLP).</abstract>
<identifier type="citekey">hou-etal-2024-course-shared</identifier>
<location>
<url>https://aclanthology.org/2024.teachingnlp-1.11</url>
</location>
<part>
<date>2024-08</date>
<extent unit="page">
<start>77</start>
<end>80</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T A Course Shared Task on Evaluating LLM Output for Clinical Questions
%A Hou, Yufang
%A Tran, Thy
%A Vu, Doan
%A Cao, Yiwen
%A Li, Kai
%A Rohde, Lukas
%A Gurevych, Iryna
%Y Al-azzawi, Sana
%Y Biester, Laura
%Y Kovács, György
%Y Marasović, Ana
%Y Mathur, Leena
%Y Mieskes, Margot
%Y Weissweiler, Leonie
%S Proceedings of the Sixth Workshop on Teaching NLP
%D 2024
%8 August
%I Association for Computational Linguistics
%C Bangkok, Thailand
%F hou-etal-2024-course-shared
%X This paper presents a shared task that we organized at the Foundations of Language Technology (FoLT) course in 2023/2024 at the Technical University of Darmstadt, which focuses on evaluating the output of Large Language Models (LLMs) in generating harmful answers to health-related clinical questions. We describe the task design considerations and report the feedback we received from the students. We expect the task and the findings reported in this paper to be relevant for instructors teaching natural language processing (NLP).
%U https://aclanthology.org/2024.teachingnlp-1.11
%P 77-80
Markdown (Informal)
[A Course Shared Task on Evaluating LLM Output for Clinical Questions](https://aclanthology.org/2024.teachingnlp-1.11) (Hou et al., TeachingNLP-WS 2024)
ACL