@inproceedings{denny-2026-lost,
title = "Lost In Translation: Uneven Grounding Degradation Under Cross-Lingual Shift in {LLM}-Assisted Cyber Threat Intelligence",
author = "Denny, Lailah",
editor = "Wang, Yimu and
Choi, Yee Man and
Wu, Di and
Ouyang, Siqi and
Azadani, Mozhgan Nasr and
Fung, Yi R. (May)",
booktitle = "Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently ({G}round{LM} 2026)",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.groundlm-1.40/",
pages = "505--512",
ISBN = "979-8-89176-478-1",
abstract = "We introduce \textbf{CrossCTI}, a multilingual benchmark for evaluating retrieval-augmented generation (RAG) grounding faithfulness in open-source large language models applied to cyber threat intelligence (CTI) analysis. Existing CTI and forensic NLP benchmarks are predominantly English-centric, leaving a critical gap in understanding how grounding faithfulness degrades under cross-lingual distribution shift, a realistic condition in multinational security operations centers. CrossCTI translates 50 English CTI reports into Arabic, Chinese, Swahili, and Russian using NLLB-200 with entity-preserving masking, achieving 100{\%} IOC preservation across languages. We construct 26 forensic question-answer pairs and evaluate each against all five language corpora and three open-source models (Qwen2.5-7B, Llama-3.1-8B, and Mistral-7B-v0.3), producing 390 evaluation instances, scored with token-level F1, character-level F1, and entailment-based attribution. Using a repeated-measures mixed-effects analysis that accounts for the fact that each of the 26 items is evaluated under all 15 model$\times$language conditions, we find that cross-lingual degradation is severe, but \textit{uneven across metrics}: language is the dominant, robust predictor of lexical (token-level) grounding quality ($p<0.001$), with no detectable effect of model identity on this metric. In contrast, model identity is a significant predictor of both character-level overlap and entailment-based attribution ($p<0.01$), including a specific interaction in one model (Mistral) that underperforms specifically in Swahili. Token-level F1 drops from 0.44 (English) to 0.03 (Arabic); gold-chunk retrieval recall drops from 0.962 (English) to 0.115 (Arabic); and the instance-level correlation between retrieval success and token-level grounding is $r=0.60$. Performance also improves from the top-$k=1$ to the top-$k=3$ retrieved passages before plateauing, with no consistent gains from the additional retrieved content. We further show that the entailment-based attribution metric is significantly elevated specifically for the model used to generate the benchmark{'}s own QA pairs, a synthetic-annotator confound that we quantify directly via within-item paired tests rather than only disclosing qualitatively. CrossCTI provides a reproducible framework for multilingual CTI grounding evaluation and a case study in how the apparent dominant factor, language or model, depends on which grounding metric is used."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="denny-2026-lost">
<titleInfo>
<title>Lost In Translation: Uneven Grounding Degradation Under Cross-Lingual Shift in LLM-Assisted Cyber Threat Intelligence</title>
</titleInfo>
<name type="personal">
<namePart type="given">Lailah</namePart>
<namePart type="family">Denny</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yimu</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yee</namePart>
<namePart type="given">Man</namePart>
<namePart type="family">Choi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Di</namePart>
<namePart type="family">Wu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siqi</namePart>
<namePart type="family">Ouyang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mozhgan</namePart>
<namePart type="given">Nasr</namePart>
<namePart type="family">Azadani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yi</namePart>
<namePart type="given">R</namePart>
<namePart type="given">(May)</namePart>
<namePart type="family">Fung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-478-1</identifier>
</relatedItem>
<abstract>We introduce CrossCTI, a multilingual benchmark for evaluating retrieval-augmented generation (RAG) grounding faithfulness in open-source large language models applied to cyber threat intelligence (CTI) analysis. Existing CTI and forensic NLP benchmarks are predominantly English-centric, leaving a critical gap in understanding how grounding faithfulness degrades under cross-lingual distribution shift, a realistic condition in multinational security operations centers. CrossCTI translates 50 English CTI reports into Arabic, Chinese, Swahili, and Russian using NLLB-200 with entity-preserving masking, achieving 100% IOC preservation across languages. We construct 26 forensic question-answer pairs and evaluate each against all five language corpora and three open-source models (Qwen2.5-7B, Llama-3.1-8B, and Mistral-7B-v0.3), producing 390 evaluation instances, scored with token-level F1, character-level F1, and entailment-based attribution. Using a repeated-measures mixed-effects analysis that accounts for the fact that each of the 26 items is evaluated under all 15 model\timeslanguage conditions, we find that cross-lingual degradation is severe, but uneven across metrics: language is the dominant, robust predictor of lexical (token-level) grounding quality (p<0.001), with no detectable effect of model identity on this metric. In contrast, model identity is a significant predictor of both character-level overlap and entailment-based attribution (p<0.01), including a specific interaction in one model (Mistral) that underperforms specifically in Swahili. Token-level F1 drops from 0.44 (English) to 0.03 (Arabic); gold-chunk retrieval recall drops from 0.962 (English) to 0.115 (Arabic); and the instance-level correlation between retrieval success and token-level grounding is r=0.60. Performance also improves from the top-k=1 to the top-k=3 retrieved passages before plateauing, with no consistent gains from the additional retrieved content. We further show that the entailment-based attribution metric is significantly elevated specifically for the model used to generate the benchmark’s own QA pairs, a synthetic-annotator confound that we quantify directly via within-item paired tests rather than only disclosing qualitatively. CrossCTI provides a reproducible framework for multilingual CTI grounding evaluation and a case study in how the apparent dominant factor, language or model, depends on which grounding metric is used.</abstract>
<identifier type="citekey">denny-2026-lost</identifier>
<location>
<url>https://aclanthology.org/2026.groundlm-1.40/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>505</start>
<end>512</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Lost In Translation: Uneven Grounding Degradation Under Cross-Lingual Shift in LLM-Assisted Cyber Threat Intelligence
%A Denny, Lailah
%Y Wang, Yimu
%Y Choi, Yee Man
%Y Wu, Di
%Y Ouyang, Siqi
%Y Azadani, Mozhgan Nasr
%Y Fung, Yi R. (May)
%S Proceedings of the 1st Workshop on Grounding Language Models: Learning Faithfully and Efficiently (GroundLM 2026)
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-478-1
%F denny-2026-lost
%X We introduce CrossCTI, a multilingual benchmark for evaluating retrieval-augmented generation (RAG) grounding faithfulness in open-source large language models applied to cyber threat intelligence (CTI) analysis. Existing CTI and forensic NLP benchmarks are predominantly English-centric, leaving a critical gap in understanding how grounding faithfulness degrades under cross-lingual distribution shift, a realistic condition in multinational security operations centers. CrossCTI translates 50 English CTI reports into Arabic, Chinese, Swahili, and Russian using NLLB-200 with entity-preserving masking, achieving 100% IOC preservation across languages. We construct 26 forensic question-answer pairs and evaluate each against all five language corpora and three open-source models (Qwen2.5-7B, Llama-3.1-8B, and Mistral-7B-v0.3), producing 390 evaluation instances, scored with token-level F1, character-level F1, and entailment-based attribution. Using a repeated-measures mixed-effects analysis that accounts for the fact that each of the 26 items is evaluated under all 15 model\timeslanguage conditions, we find that cross-lingual degradation is severe, but uneven across metrics: language is the dominant, robust predictor of lexical (token-level) grounding quality (p<0.001), with no detectable effect of model identity on this metric. In contrast, model identity is a significant predictor of both character-level overlap and entailment-based attribution (p<0.01), including a specific interaction in one model (Mistral) that underperforms specifically in Swahili. Token-level F1 drops from 0.44 (English) to 0.03 (Arabic); gold-chunk retrieval recall drops from 0.962 (English) to 0.115 (Arabic); and the instance-level correlation between retrieval success and token-level grounding is r=0.60. Performance also improves from the top-k=1 to the top-k=3 retrieved passages before plateauing, with no consistent gains from the additional retrieved content. We further show that the entailment-based attribution metric is significantly elevated specifically for the model used to generate the benchmark’s own QA pairs, a synthetic-annotator confound that we quantify directly via within-item paired tests rather than only disclosing qualitatively. CrossCTI provides a reproducible framework for multilingual CTI grounding evaluation and a case study in how the apparent dominant factor, language or model, depends on which grounding metric is used.
%U https://aclanthology.org/2026.groundlm-1.40/
%P 505-512
Markdown (Informal)
[Lost In Translation: Uneven Grounding Degradation Under Cross-Lingual Shift in LLM-Assisted Cyber Threat Intelligence](https://aclanthology.org/2026.groundlm-1.40/) (Denny, GroundLM 2026)
ACL