@inproceedings{shaik-etal-2026-office,
title = "Office Comprehension Benchmark",
author = "Shaik, Firoz and
Gomes, Mateus Pican{\c{c}}o Lima and
Aumi, Md Tanvir Islam and
Wang, Jingci and
Milunovic, Milos and
Jovanovic, Ivana and
Rathi, Jay and
Suryanarayanan, Vishwas and
Kenkare, Neha Nandan and
Xie, Weiyao and
Han, Zhipeng and
Zhang, Zheng and
Shahid, Waleed and
Nguyen, Thong Q. and
Bentley, Michael and
Stankovic, Tamara and
Chakravarthy, Rasika and
Chowdhary, Vishal",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2308/",
pages = "49518--49549",
ISBN = "979-8-89176-462-0",
abstract = "We introduce Office Comprehension Bench (OCB), the first public benchmark to jointly evaluate LLM systems on Word, Excel, and PowerPoint comprehension over native file formats (.docx, .xlsx, .pptx) and their variants. OCB consists of two tracks. File Fidelity Q A tests structural and visual perception of office artifacts {---} tables, charts, embedded images, formulas, and app-specific elements such as headers, speaker notes, and named ranges. Domain Q A tests expert-level reasoning grounded in real-world industry documents across 12 professional domains, with queries requiring multi-step analysis and synthesis across documents. Each reference answer is decomposed into atomic, binary-gradable claims, and an ensemble of LLM judges scores responses against each claim independently. Even the strongest frontier system in its default reasoning mode reaches only {\textasciitilde}59.3{\%} on Domain Q A; increasing thinking depth within a tier does not move performance materially, while moving to a higher product tier yields modest gains. We release the dataset, evaluation tooling, judge prompt, and a public leaderboard."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="shaik-etal-2026-office">
<titleInfo>
<title>Office Comprehension Benchmark</title>
</titleInfo>
<name type="personal">
<namePart type="given">Firoz</namePart>
<namePart type="family">Shaik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mateus</namePart>
<namePart type="given">Picanço</namePart>
<namePart type="given">Lima</namePart>
<namePart type="family">Gomes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Md</namePart>
<namePart type="given">Tanvir</namePart>
<namePart type="given">Islam</namePart>
<namePart type="family">Aumi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingci</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Milos</namePart>
<namePart type="family">Milunovic</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ivana</namePart>
<namePart type="family">Jovanovic</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jay</namePart>
<namePart type="family">Rathi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vishwas</namePart>
<namePart type="family">Suryanarayanan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Neha</namePart>
<namePart type="given">Nandan</namePart>
<namePart type="family">Kenkare</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Weiyao</namePart>
<namePart type="family">Xie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhipeng</namePart>
<namePart type="family">Han</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zheng</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Waleed</namePart>
<namePart type="family">Shahid</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thong</namePart>
<namePart type="given">Q</namePart>
<namePart type="family">Nguyen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Michael</namePart>
<namePart type="family">Bentley</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamara</namePart>
<namePart type="family">Stankovic</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rasika</namePart>
<namePart type="family">Chakravarthy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vishal</namePart>
<namePart type="family">Chowdhary</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>We introduce Office Comprehension Bench (OCB), the first public benchmark to jointly evaluate LLM systems on Word, Excel, and PowerPoint comprehension over native file formats (.docx, .xlsx, .pptx) and their variants. OCB consists of two tracks. File Fidelity Q A tests structural and visual perception of office artifacts — tables, charts, embedded images, formulas, and app-specific elements such as headers, speaker notes, and named ranges. Domain Q A tests expert-level reasoning grounded in real-world industry documents across 12 professional domains, with queries requiring multi-step analysis and synthesis across documents. Each reference answer is decomposed into atomic, binary-gradable claims, and an ensemble of LLM judges scores responses against each claim independently. Even the strongest frontier system in its default reasoning mode reaches only ~59.3% on Domain Q A; increasing thinking depth within a tier does not move performance materially, while moving to a higher product tier yields modest gains. We release the dataset, evaluation tooling, judge prompt, and a public leaderboard.</abstract>
<identifier type="citekey">shaik-etal-2026-office</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2308/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>49518</start>
<end>49549</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Office Comprehension Benchmark
%A Shaik, Firoz
%A Gomes, Mateus Picanço Lima
%A Aumi, Md Tanvir Islam
%A Wang, Jingci
%A Milunovic, Milos
%A Jovanovic, Ivana
%A Rathi, Jay
%A Suryanarayanan, Vishwas
%A Kenkare, Neha Nandan
%A Xie, Weiyao
%A Han, Zhipeng
%A Zhang, Zheng
%A Shahid, Waleed
%A Nguyen, Thong Q.
%A Bentley, Michael
%A Stankovic, Tamara
%A Chakravarthy, Rasika
%A Chowdhary, Vishal
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F shaik-etal-2026-office
%X We introduce Office Comprehension Bench (OCB), the first public benchmark to jointly evaluate LLM systems on Word, Excel, and PowerPoint comprehension over native file formats (.docx, .xlsx, .pptx) and their variants. OCB consists of two tracks. File Fidelity Q A tests structural and visual perception of office artifacts — tables, charts, embedded images, formulas, and app-specific elements such as headers, speaker notes, and named ranges. Domain Q A tests expert-level reasoning grounded in real-world industry documents across 12 professional domains, with queries requiring multi-step analysis and synthesis across documents. Each reference answer is decomposed into atomic, binary-gradable claims, and an ensemble of LLM judges scores responses against each claim independently. Even the strongest frontier system in its default reasoning mode reaches only ~59.3% on Domain Q A; increasing thinking depth within a tier does not move performance materially, while moving to a higher product tier yields modest gains. We release the dataset, evaluation tooling, judge prompt, and a public leaderboard.
%U https://aclanthology.org/2026.findings-emnlp.2308/
%P 49518-49549
Markdown (Informal)
[Office Comprehension Benchmark](https://aclanthology.org/2026.findings-emnlp.2308/) (Shaik et al., Findings 2026)
ACL
- Firoz Shaik, Mateus Picanço Lima Gomes, Md Tanvir Islam Aumi, Jingci Wang, Milos Milunovic, Ivana Jovanovic, Jay Rathi, Vishwas Suryanarayanan, Neha Nandan Kenkare, Weiyao Xie, Zhipeng Han, Zheng Zhang, Waleed Shahid, Thong Q. Nguyen, Michael Bentley, Tamara Stankovic, Rasika Chakravarthy, and Vishal Chowdhary. 2026. Office Comprehension Benchmark. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 49518–49549, Budapest, Hungary. Association for Computational Linguistics.