@inproceedings{jian-etal-2026-forge,
title = "{FORGE}: Fine-grained multimodal evaluation for manufacturing quality assurance",
author = "Jian, Xiangru and
Xu, Hao and
Pang, Wei and
Zhao, Xinjian and
Tao, Chengyu and
Zhang, Qixin and
Zhang, Xikun and
Zhang, Chao and
Deng, Guanzhi and
Xue, Jingxi and
Du, Juan and
Yu, Tianshu and
Tarr, Garth and
Song, Linqi and
Sun, Qiuzhuang and
Tao, Dacheng",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.888/",
pages = "18762--18839",
ISBN = "979-8-89176-462-0",
abstract = "The manufacturing sector is increasingly adopting Multimodal Large Language Models (MLLMs) to move from low-level perception toward higher-level reasoning and decision support, yet current evaluations rarely reflect the fine-grained precision that real-world manufacturing demands. Progress is further hindered by data scarcity and a lack of fine-grained domain semantics in existing datasets. To bridge this gap, we introduce FORGE, a benchmark for three core quality-assurance tasks of discrete manufacturing: workpiece verification, structural surface inspection, and assembly verification. We first construct a high-quality multimodal dataset spanning real-world 2D images and 3D point clouds, annotated with fine-grained domain semantics such as standardized model numbers. We then evaluate 18 state-of-the-art MLLMs on these three tasks, revealing significant performance gaps. Counter to conventional understanding, our bottleneck analysis on five representative models indicates that visual grounding is not the primary limiting factor. Instead, insufficient domain-specific knowledge is the key bottleneck. Beyond evaluation, we show that our structured annotations can serve as an actionable training resource: supervised fine-tuning of a compact 3B-parameter model on our data yields up to 90.8{\%} relative improvement in accuracy on held-out manufacturing scenarios, providing preliminary evidence for a practical pathway toward domain-adapted manufacturing MLLMs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="jian-etal-2026-forge">
<titleInfo>
<title>FORGE: Fine-grained multimodal evaluation for manufacturing quality assurance</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xiangru</namePart>
<namePart type="family">Jian</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hao</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wei</namePart>
<namePart type="family">Pang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xinjian</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chengyu</namePart>
<namePart type="family">Tao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qixin</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xikun</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chao</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Guanzhi</namePart>
<namePart type="family">Deng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingxi</namePart>
<namePart type="family">Xue</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juan</namePart>
<namePart type="family">Du</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tianshu</namePart>
<namePart type="family">Yu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Garth</namePart>
<namePart type="family">Tarr</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Linqi</namePart>
<namePart type="family">Song</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qiuzhuang</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dacheng</namePart>
<namePart type="family">Tao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>The manufacturing sector is increasingly adopting Multimodal Large Language Models (MLLMs) to move from low-level perception toward higher-level reasoning and decision support, yet current evaluations rarely reflect the fine-grained precision that real-world manufacturing demands. Progress is further hindered by data scarcity and a lack of fine-grained domain semantics in existing datasets. To bridge this gap, we introduce FORGE, a benchmark for three core quality-assurance tasks of discrete manufacturing: workpiece verification, structural surface inspection, and assembly verification. We first construct a high-quality multimodal dataset spanning real-world 2D images and 3D point clouds, annotated with fine-grained domain semantics such as standardized model numbers. We then evaluate 18 state-of-the-art MLLMs on these three tasks, revealing significant performance gaps. Counter to conventional understanding, our bottleneck analysis on five representative models indicates that visual grounding is not the primary limiting factor. Instead, insufficient domain-specific knowledge is the key bottleneck. Beyond evaluation, we show that our structured annotations can serve as an actionable training resource: supervised fine-tuning of a compact 3B-parameter model on our data yields up to 90.8% relative improvement in accuracy on held-out manufacturing scenarios, providing preliminary evidence for a practical pathway toward domain-adapted manufacturing MLLMs.</abstract>
<identifier type="citekey">jian-etal-2026-forge</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.888/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>18762</start>
<end>18839</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T FORGE: Fine-grained multimodal evaluation for manufacturing quality assurance
%A Jian, Xiangru
%A Xu, Hao
%A Pang, Wei
%A Zhao, Xinjian
%A Tao, Chengyu
%A Zhang, Qixin
%A Zhang, Xikun
%A Zhang, Chao
%A Deng, Guanzhi
%A Xue, Jingxi
%A Du, Juan
%A Yu, Tianshu
%A Tarr, Garth
%A Song, Linqi
%A Sun, Qiuzhuang
%A Tao, Dacheng
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F jian-etal-2026-forge
%X The manufacturing sector is increasingly adopting Multimodal Large Language Models (MLLMs) to move from low-level perception toward higher-level reasoning and decision support, yet current evaluations rarely reflect the fine-grained precision that real-world manufacturing demands. Progress is further hindered by data scarcity and a lack of fine-grained domain semantics in existing datasets. To bridge this gap, we introduce FORGE, a benchmark for three core quality-assurance tasks of discrete manufacturing: workpiece verification, structural surface inspection, and assembly verification. We first construct a high-quality multimodal dataset spanning real-world 2D images and 3D point clouds, annotated with fine-grained domain semantics such as standardized model numbers. We then evaluate 18 state-of-the-art MLLMs on these three tasks, revealing significant performance gaps. Counter to conventional understanding, our bottleneck analysis on five representative models indicates that visual grounding is not the primary limiting factor. Instead, insufficient domain-specific knowledge is the key bottleneck. Beyond evaluation, we show that our structured annotations can serve as an actionable training resource: supervised fine-tuning of a compact 3B-parameter model on our data yields up to 90.8% relative improvement in accuracy on held-out manufacturing scenarios, providing preliminary evidence for a practical pathway toward domain-adapted manufacturing MLLMs.
%U https://aclanthology.org/2026.findings-emnlp.888/
%P 18762-18839
Markdown (Informal)
[FORGE: Fine-grained multimodal evaluation for manufacturing quality assurance](https://aclanthology.org/2026.findings-emnlp.888/) (Jian et al., Findings 2026)
ACL
- Xiangru Jian, Hao Xu, Wei Pang, Xinjian Zhao, Chengyu Tao, Qixin Zhang, Xikun Zhang, Chao Zhang, Guanzhi Deng, Jingxi Xue, Juan Du, Tianshu Yu, Garth Tarr, Linqi Song, Qiuzhuang Sun, and Dacheng Tao. 2026. FORGE: Fine-grained multimodal evaluation for manufacturing quality assurance. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 18762–18839, Budapest, Hungary. Association for Computational Linguistics.