@inproceedings{karimi-etal-2026-issue,
title = "Issue Detection and Category Classification in Domain-Specific Technical Logbooks",
author = "Karimi, Afshin and
Hartl, Ingmar and
Tuennermann, Henrik and
Lauscher, Anne",
editor = "Piperidis, Stelios and
Bel, N{\'u}ria and
van den Heuvel, Henk and
Ide, Nancy and
Krek, Simon and
Toral, Antonio",
booktitle = "Proceedings of the Fifteenth Language Resources and Evaluation Conference",
month = may,
year = "2026",
address = "Palma de Mallorca, Spain",
publisher = "ELRA Language Resource Association",
url = "https://aclanthology.org/2026.lrec-1.476/",
doi = "10.63317/4q9gt37vbz6w",
pages = "5996--6008",
abstract = "Operating large-scale research infrastructures such as free-electron lasers produces vast amounts of operator-authored documentation that records daily observations, anomalies, and maintenance actions. These logbooks and incident reports contain valuable operational knowledge but often remain underexplored due to their unstructured, domain-specific language. While large language models (LLMs) show strong generalization in general domains, their effectiveness on such technical operator text has, to the best of our knowledge, not been systematically assessed. We introduce two new English datasets from real-world laser operations: (i) a logbook dataset annotated for binary issue detection (does an entry describe or report an actionable fault?), and (ii) an operator ticket dataset annotated for multi-class issue categorization assign each ticket to one of 13 technical categories). The corpora comprise 2,979 logbook entries and 758 tickets from 2022{--}2024; both are cleaned, anonymized, and suitable for benchmarking classification performance. We evaluate four open LLMs (LLaMA-3, Mistral-Small, Qwen-3-30B, GPT-OSS-120B) under zero-shot, few-shot, and chain-of-thought (CoT) prompting, using multiple semantically equivalent prompt variants per setting to assess robustness. Across both tasks, few-shot prompting is consistently strongest, with top systems reaching F1 approx 0.84 for logbook issue detection and Macro-F1 0.42 for operator ticket categorization. These results suggest that incorporating a handful of in-domain examples can substantially improve performance on operator-authored technical text, even without fine-tuning."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="karimi-etal-2026-issue">
<titleInfo>
<title>Issue Detection and Category Classification in Domain-Specific Technical Logbooks</title>
</titleInfo>
<name type="personal">
<namePart type="given">Afshin</namePart>
<namePart type="family">Karimi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ingmar</namePart>
<namePart type="family">Hartl</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henrik</namePart>
<namePart type="family">Tuennermann</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anne</namePart>
<namePart type="family">Lauscher</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Fifteenth Language Resources and Evaluation Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Stelios</namePart>
<namePart type="family">Piperidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Núria</namePart>
<namePart type="family">Bel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henk</namePart>
<namePart type="family">van den Heuvel</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nancy</namePart>
<namePart type="family">Ide</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Simon</namePart>
<namePart type="family">Krek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Antonio</namePart>
<namePart type="family">Toral</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>ELRA Language Resource Association</publisher>
<place>
<placeTerm type="text">Palma de Mallorca, Spain</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Operating large-scale research infrastructures such as free-electron lasers produces vast amounts of operator-authored documentation that records daily observations, anomalies, and maintenance actions. These logbooks and incident reports contain valuable operational knowledge but often remain underexplored due to their unstructured, domain-specific language. While large language models (LLMs) show strong generalization in general domains, their effectiveness on such technical operator text has, to the best of our knowledge, not been systematically assessed. We introduce two new English datasets from real-world laser operations: (i) a logbook dataset annotated for binary issue detection (does an entry describe or report an actionable fault?), and (ii) an operator ticket dataset annotated for multi-class issue categorization assign each ticket to one of 13 technical categories). The corpora comprise 2,979 logbook entries and 758 tickets from 2022–2024; both are cleaned, anonymized, and suitable for benchmarking classification performance. We evaluate four open LLMs (LLaMA-3, Mistral-Small, Qwen-3-30B, GPT-OSS-120B) under zero-shot, few-shot, and chain-of-thought (CoT) prompting, using multiple semantically equivalent prompt variants per setting to assess robustness. Across both tasks, few-shot prompting is consistently strongest, with top systems reaching F1 approx 0.84 for logbook issue detection and Macro-F1 0.42 for operator ticket categorization. These results suggest that incorporating a handful of in-domain examples can substantially improve performance on operator-authored technical text, even without fine-tuning.</abstract>
<identifier type="citekey">karimi-etal-2026-issue</identifier>
<identifier type="doi">10.63317/4q9gt37vbz6w</identifier>
<location>
<url>https://aclanthology.org/2026.lrec-1.476/</url>
</location>
<part>
<date>2026-05</date>
<extent unit="page">
<start>5996</start>
<end>6008</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Issue Detection and Category Classification in Domain-Specific Technical Logbooks
%A Karimi, Afshin
%A Hartl, Ingmar
%A Tuennermann, Henrik
%A Lauscher, Anne
%Y Piperidis, Stelios
%Y Bel, Núria
%Y van den Heuvel, Henk
%Y Ide, Nancy
%Y Krek, Simon
%Y Toral, Antonio
%S Proceedings of the Fifteenth Language Resources and Evaluation Conference
%D 2026
%8 May
%I ELRA Language Resource Association
%C Palma de Mallorca, Spain
%F karimi-etal-2026-issue
%X Operating large-scale research infrastructures such as free-electron lasers produces vast amounts of operator-authored documentation that records daily observations, anomalies, and maintenance actions. These logbooks and incident reports contain valuable operational knowledge but often remain underexplored due to their unstructured, domain-specific language. While large language models (LLMs) show strong generalization in general domains, their effectiveness on such technical operator text has, to the best of our knowledge, not been systematically assessed. We introduce two new English datasets from real-world laser operations: (i) a logbook dataset annotated for binary issue detection (does an entry describe or report an actionable fault?), and (ii) an operator ticket dataset annotated for multi-class issue categorization assign each ticket to one of 13 technical categories). The corpora comprise 2,979 logbook entries and 758 tickets from 2022–2024; both are cleaned, anonymized, and suitable for benchmarking classification performance. We evaluate four open LLMs (LLaMA-3, Mistral-Small, Qwen-3-30B, GPT-OSS-120B) under zero-shot, few-shot, and chain-of-thought (CoT) prompting, using multiple semantically equivalent prompt variants per setting to assess robustness. Across both tasks, few-shot prompting is consistently strongest, with top systems reaching F1 approx 0.84 for logbook issue detection and Macro-F1 0.42 for operator ticket categorization. These results suggest that incorporating a handful of in-domain examples can substantially improve performance on operator-authored technical text, even without fine-tuning.
%R 10.63317/4q9gt37vbz6w
%U https://aclanthology.org/2026.lrec-1.476/
%U https://doi.org/10.63317/4q9gt37vbz6w
%P 5996-6008
Markdown (Informal)
[Issue Detection and Category Classification in Domain-Specific Technical Logbooks](https://aclanthology.org/2026.lrec-1.476/) (Karimi et al., LREC 2026)
ACL