@inproceedings{flicke-etal-2026-horizonbench,
title = "{H}orizon{B}ench: Evaluating Ideation Agents on Future Scientific Discoveries",
author = "Flicke, Markus and
He, Haoyu and
Chitta, Kashyap and
Cao, Yong and
Geiger, Andreas",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the {A}sia-Pacific Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.aacl-long.202/",
pages = "4466--4479",
ISBN = "979-8-89176-439-2",
abstract = {As ideation agents become more capable, evaluating them poses an increasing challenge. LLM-as-a-judge approaches are brittle and gameable, while human evaluation is expensive and hard to scale. To address this, we introduce HorizonBench, an evaluation methodology leveraging future scientific discoveries. HorizonBench uses published papers as ground truth for ideation quality. Specifically, ideation agents pretraining and context retrieval data are strictly constrained to pre-publication knowledge, and scored on how well their predicted ideas align with subsequently published work. Using HorizonBench, we conduct the first unified comparison of five published ideation agents across multiple frontier model backbones. We verify that our soft recall metric captures meaningful ideation quality aligned to human judgement via a user study with 51 domain experts. We then uncover a surprising finding: a na{\"i}ve baseline that receives only the research goal as an input consistently matches or outperforms all existing agents that require retrieval or multi-step reasoning, highlighting the importance of the research goal for ideation. Based on this result, we develop OSCAR, an ideation agent that distils retrieved papers into insights and explicitly reinforces the research goal in each step of the pipeline. OSCAR outperforms all evaluated agents on HorizonBench. We publish the code and an online version of HorizonBench.}
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="flicke-etal-2026-horizonbench">
<titleInfo>
<title>HorizonBench: Evaluating Ideation Agents on Future Scientific Discoveries</title>
</titleInfo>
<name type="personal">
<namePart type="given">Markus</namePart>
<namePart type="family">Flicke</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haoyu</namePart>
<namePart type="family">He</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kashyap</namePart>
<namePart type="family">Chitta</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yong</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Geiger</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-439-2</identifier>
</relatedItem>
<abstract>As ideation agents become more capable, evaluating them poses an increasing challenge. LLM-as-a-judge approaches are brittle and gameable, while human evaluation is expensive and hard to scale. To address this, we introduce HorizonBench, an evaluation methodology leveraging future scientific discoveries. HorizonBench uses published papers as ground truth for ideation quality. Specifically, ideation agents pretraining and context retrieval data are strictly constrained to pre-publication knowledge, and scored on how well their predicted ideas align with subsequently published work. Using HorizonBench, we conduct the first unified comparison of five published ideation agents across multiple frontier model backbones. We verify that our soft recall metric captures meaningful ideation quality aligned to human judgement via a user study with 51 domain experts. We then uncover a surprising finding: a naïve baseline that receives only the research goal as an input consistently matches or outperforms all existing agents that require retrieval or multi-step reasoning, highlighting the importance of the research goal for ideation. Based on this result, we develop OSCAR, an ideation agent that distils retrieved papers into insights and explicitly reinforces the research goal in each step of the pipeline. OSCAR outperforms all evaluated agents on HorizonBench. We publish the code and an online version of HorizonBench.</abstract>
<identifier type="citekey">flicke-etal-2026-horizonbench</identifier>
<location>
<url>https://aclanthology.org/2026.aacl-long.202/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>4466</start>
<end>4479</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T HorizonBench: Evaluating Ideation Agents on Future Scientific Discoveries
%A Flicke, Markus
%A He, Haoyu
%A Chitta, Kashyap
%A Cao, Yong
%A Geiger, Andreas
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-439-2
%F flicke-etal-2026-horizonbench
%X As ideation agents become more capable, evaluating them poses an increasing challenge. LLM-as-a-judge approaches are brittle and gameable, while human evaluation is expensive and hard to scale. To address this, we introduce HorizonBench, an evaluation methodology leveraging future scientific discoveries. HorizonBench uses published papers as ground truth for ideation quality. Specifically, ideation agents pretraining and context retrieval data are strictly constrained to pre-publication knowledge, and scored on how well their predicted ideas align with subsequently published work. Using HorizonBench, we conduct the first unified comparison of five published ideation agents across multiple frontier model backbones. We verify that our soft recall metric captures meaningful ideation quality aligned to human judgement via a user study with 51 domain experts. We then uncover a surprising finding: a naïve baseline that receives only the research goal as an input consistently matches or outperforms all existing agents that require retrieval or multi-step reasoning, highlighting the importance of the research goal for ideation. Based on this result, we develop OSCAR, an ideation agent that distils retrieved papers into insights and explicitly reinforces the research goal in each step of the pipeline. OSCAR outperforms all evaluated agents on HorizonBench. We publish the code and an online version of HorizonBench.
%U https://aclanthology.org/2026.aacl-long.202/
%P 4466-4479
Markdown (Informal)
[HorizonBench: Evaluating Ideation Agents on Future Scientific Discoveries](https://aclanthology.org/2026.aacl-long.202/) (Flicke et al., AACL-IJCNLP 2026)
ACL
- Markus Flicke, Haoyu He, Kashyap Chitta, Yong Cao, and Andreas Geiger. 2026. HorizonBench: Evaluating Ideation Agents on Future Scientific Discoveries. In Proceedings of the 15th International Joint Conference on Natural Language Processing and the 5th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), pages 4466–4479, Hengqin, China. Association for Computational Linguistics.