@inproceedings{alshaikh-alrehiely-2026-llm,
title = "Why {LLM} Aggregation Falls Short: Correlated Errors and Selection Gaps in {A}rabic Punctuation Restoration",
author = "Alshaikh, Rana and
Alrehiely, Majedah",
editor = "Abdul-Mageed, Muhammad and
Elsayed, Tamer and
Bouamor, Houda and
El-Beltagy, Samhaa and
Al-Khalifa, Hend and
Abdelali, Ahmed and
Elmadany, AbdelRahim and
El Mekki, Abdellah and
Darwish, Kareem and
Alkhereyf, Sakhar and
Luqman, Hamzah and
Khalifa, Salam and
Magdy, Samar and
Alyafeai, Zaid and
Hamed, Injy and
Zaghouani, Wajdi and
Zitouni, Imed",
booktitle = "Proceedings of The Fourth {A}rabic Natural Language Processing Conference",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.arabicnlp-main.36/",
pages = "522--537",
ISBN = "979-8-89176-473-6",
abstract = "Combining LLMs for Arabic punctuation restoration should help when their errors are complementary, but aggregation failure can reflect either missing coverage, where no predictor proposes the correct label, or failed selection, where the correct label is present but not selected. We investigate this distinction with vote-based, text-blind aggregators that use predictor labels, agreement, position, and neighbouring predictions, but not Arabic words. We evaluate three frontier LLMs and a fine-tuned AraBERT encoder on AraPunc-sub and ABC. The LLMs show strongly correlated errors: when all three are wrong, they choose the same incorrect label in 79.4{\%} of cases on AraPunc-sub and 77.8{\%} on ABC. Yet the four-predictor oracle achieves macro-$F_1$ scores of 0.857 and 0.778, surpassing the best standalone predictors (0.639 and 0.496). Leave-one-predictor-out analysis shows that AraBERT supplies the largest unique oracle coverage. Standard aggregation recovers only part of this headroom, and no tested aggregator outperforms standalone AraBERT on AraPunc-sub. A selectability probe confirms that some covered disagreements are learnable, but a deployable router offers only modest, corpus-dependent gains. Even a text-aware peer-deliberation step deepens rather than breaks the correlation, as weaker models conform to their peers. Hence, in the tested text-blind regime, the bottleneck is primarily selection rather than coverage, without ruling out text-aware selectors."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="alshaikh-alrehiely-2026-llm">
<titleInfo>
<title>Why LLM Aggregation Falls Short: Correlated Errors and Selection Gaps in Arabic Punctuation Restoration</title>
</titleInfo>
<name type="personal">
<namePart type="given">Rana</namePart>
<namePart type="family">Alshaikh</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Majedah</namePart>
<namePart type="family">Alrehiely</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of The Fourth Arabic Natural Language Processing Conference</title>
</titleInfo>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="family">Abdul-Mageed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tamer</namePart>
<namePart type="family">Elsayed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samhaa</namePart>
<namePart type="family">El-Beltagy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hend</namePart>
<namePart type="family">Al-Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ahmed</namePart>
<namePart type="family">Abdelali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">AbdelRahim</namePart>
<namePart type="family">Elmadany</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Abdellah</namePart>
<namePart type="family">El Mekki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kareem</namePart>
<namePart type="family">Darwish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sakhar</namePart>
<namePart type="family">Alkhereyf</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hamzah</namePart>
<namePart type="family">Luqman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Salam</namePart>
<namePart type="family">Khalifa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Samar</namePart>
<namePart type="family">Magdy</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zaid</namePart>
<namePart type="family">Alyafeai</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Injy</namePart>
<namePart type="family">Hamed</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wajdi</namePart>
<namePart type="family">Zaghouani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Imed</namePart>
<namePart type="family">Zitouni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-473-6</identifier>
</relatedItem>
<abstract>Combining LLMs for Arabic punctuation restoration should help when their errors are complementary, but aggregation failure can reflect either missing coverage, where no predictor proposes the correct label, or failed selection, where the correct label is present but not selected. We investigate this distinction with vote-based, text-blind aggregators that use predictor labels, agreement, position, and neighbouring predictions, but not Arabic words. We evaluate three frontier LLMs and a fine-tuned AraBERT encoder on AraPunc-sub and ABC. The LLMs show strongly correlated errors: when all three are wrong, they choose the same incorrect label in 79.4% of cases on AraPunc-sub and 77.8% on ABC. Yet the four-predictor oracle achieves macro-F₁ scores of 0.857 and 0.778, surpassing the best standalone predictors (0.639 and 0.496). Leave-one-predictor-out analysis shows that AraBERT supplies the largest unique oracle coverage. Standard aggregation recovers only part of this headroom, and no tested aggregator outperforms standalone AraBERT on AraPunc-sub. A selectability probe confirms that some covered disagreements are learnable, but a deployable router offers only modest, corpus-dependent gains. Even a text-aware peer-deliberation step deepens rather than breaks the correlation, as weaker models conform to their peers. Hence, in the tested text-blind regime, the bottleneck is primarily selection rather than coverage, without ruling out text-aware selectors.</abstract>
<identifier type="citekey">alshaikh-alrehiely-2026-llm</identifier>
<location>
<url>https://aclanthology.org/2026.arabicnlp-main.36/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>522</start>
<end>537</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Why LLM Aggregation Falls Short: Correlated Errors and Selection Gaps in Arabic Punctuation Restoration
%A Alshaikh, Rana
%A Alrehiely, Majedah
%Y Abdul-Mageed, Muhammad
%Y Elsayed, Tamer
%Y Bouamor, Houda
%Y El-Beltagy, Samhaa
%Y Al-Khalifa, Hend
%Y Abdelali, Ahmed
%Y Elmadany, AbdelRahim
%Y El Mekki, Abdellah
%Y Darwish, Kareem
%Y Alkhereyf, Sakhar
%Y Luqman, Hamzah
%Y Khalifa, Salam
%Y Magdy, Samar
%Y Alyafeai, Zaid
%Y Hamed, Injy
%Y Zaghouani, Wajdi
%Y Zitouni, Imed
%S Proceedings of The Fourth Arabic Natural Language Processing Conference
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-473-6
%F alshaikh-alrehiely-2026-llm
%X Combining LLMs for Arabic punctuation restoration should help when their errors are complementary, but aggregation failure can reflect either missing coverage, where no predictor proposes the correct label, or failed selection, where the correct label is present but not selected. We investigate this distinction with vote-based, text-blind aggregators that use predictor labels, agreement, position, and neighbouring predictions, but not Arabic words. We evaluate three frontier LLMs and a fine-tuned AraBERT encoder on AraPunc-sub and ABC. The LLMs show strongly correlated errors: when all three are wrong, they choose the same incorrect label in 79.4% of cases on AraPunc-sub and 77.8% on ABC. Yet the four-predictor oracle achieves macro-F₁ scores of 0.857 and 0.778, surpassing the best standalone predictors (0.639 and 0.496). Leave-one-predictor-out analysis shows that AraBERT supplies the largest unique oracle coverage. Standard aggregation recovers only part of this headroom, and no tested aggregator outperforms standalone AraBERT on AraPunc-sub. A selectability probe confirms that some covered disagreements are learnable, but a deployable router offers only modest, corpus-dependent gains. Even a text-aware peer-deliberation step deepens rather than breaks the correlation, as weaker models conform to their peers. Hence, in the tested text-blind regime, the bottleneck is primarily selection rather than coverage, without ruling out text-aware selectors.
%U https://aclanthology.org/2026.arabicnlp-main.36/
%P 522-537
Markdown (Informal)
[Why LLM Aggregation Falls Short: Correlated Errors and Selection Gaps in Arabic Punctuation Restoration](https://aclanthology.org/2026.arabicnlp-main.36/) (Alshaikh & Alrehiely, ArabicNLP 2026)
ACL