@inproceedings{lee-etal-2026-scr,
title = "{SCR}-Eval: Evaluating Video Subtitle Translation with a Script-like Context Representation",
author = "Lee, Jongin and
Hwang, Juyi and
Kim, Sehyun",
editor = "Atanasova, Pepa and
Haffari, Gholamreza and
Wang, Minghan",
booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.emnlp-industry.80/",
pages = "1339--1360",
ISBN = "979-8-89176-460-6",
abstract = "The quality of video subtitle translation cannot be assessed from sentence-level semantic equivalence alone. Subtitles are governed by audiovisual cues, speaker relationships, and narrative context that accumulates across scenes, yet existing automatic metrics and text-centric LLM-based evaluation do not reflect these factors.\par We propose SCR-Eval, a context-aware evaluation framework for video subtitle translation, built on a Script-like Context Representation (SCR) reconstructed from the video. The evaluator aligns the translated subtitles to the SCR{'}s semantic scenes, then performs MQM error analysis with an LLM judge.\par On three open movies, we benchmark five English-to-Korean subtitle providers against blind, randomized annotations by seven professional evaluators. The framework{'}s provider ranking agrees strongly with the experts (system-level Spearman {\ensuremath{\rho}} = 0.967). In ablations, agreement weakens stepwise from full context through scene-bounded text to isolated source{--}translation pairs. Automatic baselines agree less: MetricX and xCOMET correlate negatively with the expert ranking, while RUBRIC-MQM correlates only weakly.\par With full context, SCR-Eval shows substantially stronger agreement with experts than these baselines."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lee-etal-2026-scr">
<titleInfo>
<title>SCR-Eval: Evaluating Video Subtitle Translation with a Script-like Context Representation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jongin</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juyi</namePart>
<namePart type="family">Hwang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sehyun</namePart>
<namePart type="family">Kim</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pepa</namePart>
<namePart type="family">Atanasova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gholamreza</namePart>
<namePart type="family">Haffari</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-460-6</identifier>
</relatedItem>
<abstract>The quality of video subtitle translation cannot be assessed from sentence-level semantic equivalence alone. Subtitles are governed by audiovisual cues, speaker relationships, and narrative context that accumulates across scenes, yet existing automatic metrics and text-centric LLM-based evaluation do not reflect these factors.\par We propose SCR-Eval, a context-aware evaluation framework for video subtitle translation, built on a Script-like Context Representation (SCR) reconstructed from the video. The evaluator aligns the translated subtitles to the SCR’s semantic scenes, then performs MQM error analysis with an LLM judge.\par On three open movies, we benchmark five English-to-Korean subtitle providers against blind, randomized annotations by seven professional evaluators. The framework’s provider ranking agrees strongly with the experts (system-level Spearman \ensuremathρ = 0.967). In ablations, agreement weakens stepwise from full context through scene-bounded text to isolated source–translation pairs. Automatic baselines agree less: MetricX and xCOMET correlate negatively with the expert ranking, while RUBRIC-MQM correlates only weakly.\par With full context, SCR-Eval shows substantially stronger agreement with experts than these baselines.</abstract>
<identifier type="citekey">lee-etal-2026-scr</identifier>
<location>
<url>https://aclanthology.org/2026.emnlp-industry.80/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>1339</start>
<end>1360</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T SCR-Eval: Evaluating Video Subtitle Translation with a Script-like Context Representation
%A Lee, Jongin
%A Hwang, Juyi
%A Kim, Sehyun
%Y Atanasova, Pepa
%Y Haffari, Gholamreza
%Y Wang, Minghan
%S Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing: Industry Track
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-460-6
%F lee-etal-2026-scr
%X The quality of video subtitle translation cannot be assessed from sentence-level semantic equivalence alone. Subtitles are governed by audiovisual cues, speaker relationships, and narrative context that accumulates across scenes, yet existing automatic metrics and text-centric LLM-based evaluation do not reflect these factors.\par We propose SCR-Eval, a context-aware evaluation framework for video subtitle translation, built on a Script-like Context Representation (SCR) reconstructed from the video. The evaluator aligns the translated subtitles to the SCR’s semantic scenes, then performs MQM error analysis with an LLM judge.\par On three open movies, we benchmark five English-to-Korean subtitle providers against blind, randomized annotations by seven professional evaluators. The framework’s provider ranking agrees strongly with the experts (system-level Spearman \ensuremathρ = 0.967). In ablations, agreement weakens stepwise from full context through scene-bounded text to isolated source–translation pairs. Automatic baselines agree less: MetricX and xCOMET correlate negatively with the expert ranking, while RUBRIC-MQM correlates only weakly.\par With full context, SCR-Eval shows substantially stronger agreement with experts than these baselines.
%U https://aclanthology.org/2026.emnlp-industry.80/
%P 1339-1360
Markdown (Informal)
[SCR-Eval: Evaluating Video Subtitle Translation with a Script-like Context Representation](https://aclanthology.org/2026.emnlp-industry.80/) (Lee et al., EMNLP 2026)
ACL