@inproceedings{lin-chen-2026-beyond,
title = "Beyond Correctness: Benchmarking {LLM}s for Algorithmic Program Optimization",
author = "Lin, Yehua and
Chen, Yin",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.152/",
pages = "3211--3226",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) can generate functionally correct code, but their ability to optimize programs is underexplored. We introduce OEISPerf, a code-to-code benchmark for algorithmic program optimization built from alternative implementations of integer sequences in the On-Line Encyclopedia of Integer Sequences. For each sequence we mine multiple mathematical formulations, synthesize and validate Python implementations, and pair a substantially slower program with the fastest validated reference found by our construction pipeline, producing instances that often require substantial structural transformations and can yield substantial speedups. Evaluating 15 open-source and closed-source LLMs shows that even the strongest evaluated setting, Gemini-3-Flash with its thinking mode enabled, matches the reference efficiency on only 67.9{\%} of test instances. Across LLMs, we observe both incorrect ``optimizations'' that break correctness and superficial refactorings that preserve suboptimal complexity. Finally, we fine-tune Qwen2.5-Coder-3B-Instruct via reinforcement learning, markedly improving reference attainment on OEISPerf while slightly improving performance on standard code-generation benchmarks."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lin-chen-2026-beyond">
<titleInfo>
<title>Beyond Correctness: Benchmarking LLMs for Algorithmic Program Optimization</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yehua</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yin</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) can generate functionally correct code, but their ability to optimize programs is underexplored. We introduce OEISPerf, a code-to-code benchmark for algorithmic program optimization built from alternative implementations of integer sequences in the On-Line Encyclopedia of Integer Sequences. For each sequence we mine multiple mathematical formulations, synthesize and validate Python implementations, and pair a substantially slower program with the fastest validated reference found by our construction pipeline, producing instances that often require substantial structural transformations and can yield substantial speedups. Evaluating 15 open-source and closed-source LLMs shows that even the strongest evaluated setting, Gemini-3-Flash with its thinking mode enabled, matches the reference efficiency on only 67.9% of test instances. Across LLMs, we observe both incorrect “optimizations” that break correctness and superficial refactorings that preserve suboptimal complexity. Finally, we fine-tune Qwen2.5-Coder-3B-Instruct via reinforcement learning, markedly improving reference attainment on OEISPerf while slightly improving performance on standard code-generation benchmarks.</abstract>
<identifier type="citekey">lin-chen-2026-beyond</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.152/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>3211</start>
<end>3226</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Beyond Correctness: Benchmarking LLMs for Algorithmic Program Optimization
%A Lin, Yehua
%A Chen, Yin
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F lin-chen-2026-beyond
%X Large language models (LLMs) can generate functionally correct code, but their ability to optimize programs is underexplored. We introduce OEISPerf, a code-to-code benchmark for algorithmic program optimization built from alternative implementations of integer sequences in the On-Line Encyclopedia of Integer Sequences. For each sequence we mine multiple mathematical formulations, synthesize and validate Python implementations, and pair a substantially slower program with the fastest validated reference found by our construction pipeline, producing instances that often require substantial structural transformations and can yield substantial speedups. Evaluating 15 open-source and closed-source LLMs shows that even the strongest evaluated setting, Gemini-3-Flash with its thinking mode enabled, matches the reference efficiency on only 67.9% of test instances. Across LLMs, we observe both incorrect “optimizations” that break correctness and superficial refactorings that preserve suboptimal complexity. Finally, we fine-tune Qwen2.5-Coder-3B-Instruct via reinforcement learning, markedly improving reference attainment on OEISPerf while slightly improving performance on standard code-generation benchmarks.
%U https://aclanthology.org/2026.findings-emnlp.152/
%P 3211-3226
Markdown (Informal)
[Beyond Correctness: Benchmarking LLMs for Algorithmic Program Optimization](https://aclanthology.org/2026.findings-emnlp.152/) (Lin & Chen, Findings 2026)
ACL