@inproceedings{erana-etal-2026-cognac,
title = "{COGNAC} at {S}em{E}val-2026 Task 4: Evaluating Narrative Components with {LLM}s for Hard Story Similarity Cases",
author = "Erana, Tisa Islam and
Islam, Azwad Anjum and
Sharma, Anshu and
Finlayson, Mark",
editor = "Kochmar, Ekaterina and
Ghosh, Debanjan and
North, Kai and
Komachi, Mamoru",
booktitle = "Proceedings of the 20th {I}nternational {W}orkshop on {S}emantic {E}valuation (2026)",
month = jul,
year = "2026",
address = "San Diego, California, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.semeval-1.290/",
pages = "2290--2300",
ISBN = "979-8-89176-414-9",
abstract = "This paper presents a two-stage system for the SemEval-2026 shared task on narrative similarity. The task defines similarity in terms of three components: abstract theme, course of action, and outcome. For Track A, the system first applies majority voting over multiple independent large language model (LLM) judgments to handle high-agreement (easy) cases. For low-agreement (difficult) cases, it routes examples to a second stage that decomposes stories into theme, course of action, and outcome, and either (i) scores these components individually with learned weights or (ii) uses structured chain-of-thought prompting to compare stories along the three dimensions. This two-stage approach improves robustness on difficult examples and achieves first place with 0.78 test accuracy. For Track B, the system generates embeddings of full stories and of individual narrative components using several embedding models. Experiments show that embeddings derived from the course-of-action component alone yield the best performance, achieving 0.72 accuracy and ranking first. Additional analyses reveal substantial annotation variability in the dataset and highlight the importance of handling ambiguity and disagreement when modeling narrative similarity."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="erana-etal-2026-cognac">
<titleInfo>
<title>COGNAC at SemEval-2026 Task 4: Evaluating Narrative Components with LLMs for Hard Story Similarity Cases</title>
</titleInfo>
<name type="personal">
<namePart type="given">Tisa</namePart>
<namePart type="given">Islam</namePart>
<namePart type="family">Erana</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Azwad</namePart>
<namePart type="given">Anjum</namePart>
<namePart type="family">Islam</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anshu</namePart>
<namePart type="family">Sharma</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mark</namePart>
<namePart type="family">Finlayson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 20th International Workshop on Semantic Evaluation (2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ekaterina</namePart>
<namePart type="family">Kochmar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Debanjan</namePart>
<namePart type="family">Ghosh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kai</namePart>
<namePart type="family">North</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mamoru</namePart>
<namePart type="family">Komachi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">San Diego, California, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-414-9</identifier>
</relatedItem>
<abstract>This paper presents a two-stage system for the SemEval-2026 shared task on narrative similarity. The task defines similarity in terms of three components: abstract theme, course of action, and outcome. For Track A, the system first applies majority voting over multiple independent large language model (LLM) judgments to handle high-agreement (easy) cases. For low-agreement (difficult) cases, it routes examples to a second stage that decomposes stories into theme, course of action, and outcome, and either (i) scores these components individually with learned weights or (ii) uses structured chain-of-thought prompting to compare stories along the three dimensions. This two-stage approach improves robustness on difficult examples and achieves first place with 0.78 test accuracy. For Track B, the system generates embeddings of full stories and of individual narrative components using several embedding models. Experiments show that embeddings derived from the course-of-action component alone yield the best performance, achieving 0.72 accuracy and ranking first. Additional analyses reveal substantial annotation variability in the dataset and highlight the importance of handling ambiguity and disagreement when modeling narrative similarity.</abstract>
<identifier type="citekey">erana-etal-2026-cognac</identifier>
<location>
<url>https://aclanthology.org/2026.semeval-1.290/</url>
</location>
<part>
<date>2026-07</date>
<extent unit="page">
<start>2290</start>
<end>2300</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T COGNAC at SemEval-2026 Task 4: Evaluating Narrative Components with LLMs for Hard Story Similarity Cases
%A Erana, Tisa Islam
%A Islam, Azwad Anjum
%A Sharma, Anshu
%A Finlayson, Mark
%Y Kochmar, Ekaterina
%Y Ghosh, Debanjan
%Y North, Kai
%Y Komachi, Mamoru
%S Proceedings of the 20th International Workshop on Semantic Evaluation (2026)
%D 2026
%8 July
%I Association for Computational Linguistics
%C San Diego, California, USA
%@ 979-8-89176-414-9
%F erana-etal-2026-cognac
%X This paper presents a two-stage system for the SemEval-2026 shared task on narrative similarity. The task defines similarity in terms of three components: abstract theme, course of action, and outcome. For Track A, the system first applies majority voting over multiple independent large language model (LLM) judgments to handle high-agreement (easy) cases. For low-agreement (difficult) cases, it routes examples to a second stage that decomposes stories into theme, course of action, and outcome, and either (i) scores these components individually with learned weights or (ii) uses structured chain-of-thought prompting to compare stories along the three dimensions. This two-stage approach improves robustness on difficult examples and achieves first place with 0.78 test accuracy. For Track B, the system generates embeddings of full stories and of individual narrative components using several embedding models. Experiments show that embeddings derived from the course-of-action component alone yield the best performance, achieving 0.72 accuracy and ranking first. Additional analyses reveal substantial annotation variability in the dataset and highlight the importance of handling ambiguity and disagreement when modeling narrative similarity.
%U https://aclanthology.org/2026.semeval-1.290/
%P 2290-2300
Markdown (Informal)
[COGNAC at SemEval-2026 Task 4: Evaluating Narrative Components with LLMs for Hard Story Similarity Cases](https://aclanthology.org/2026.semeval-1.290/) (Erana et al., SemEval 2026)
ACL