@inproceedings{lasik-etal-2026-reassessing,
title = "Reassessing High-Performing {LLM}s on {P}olish Medical Exams: True Competence or Bias-Driven Performance?",
author = "Lasik, Antoni and
Pokrywka, Jakub and
Grzybowski, {\L}ukasz and
Kaczmarek, Jeremi Ignacy and
Korza{\'n}ska, Gabriela and
{\'S}wieczkowski-Feiz, Janusz and
Pastuszek, Oskar and
Hoffman, Paulina and
Dabrowski, Jakub Tomasz and
Kusa, Wojciech",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.2394/",
doi = "10.18653/2026.findings-emnlp.2394",
pages = "51441--51467",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) in medicine are mainly evaluated using multiple-choice question answering (MCQA), which can overestimate real clinical ability due to guessing strategies and answer biases. To address these limitations, we introduce an expanded and more challenging benchmark based on Polish medical exams, adding over 15,000 questions, two new domains, and four structural modifications that reduce MCQA-specific artifacts and better test reasoning. We evaluate 21 LLMs and show that evaluation design strongly affects results. Under our harder setup, the best model (Qwen3.5-122B) drops by 28.4 and 31 pp on English and Polish exams, respectively. Despite low evidence of data contamination, standard MCQA scores do not reliably reflect true medical competence. To facilitate further research, we make our benchmark publicly available: https://github.com/NASK-NLP/Polish-Medical-Exams"
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lasik-etal-2026-reassessing">
<titleInfo>
<title>Reassessing High-Performing LLMs on Polish Medical Exams: True Competence or Bias-Driven Performance?</title>
</titleInfo>
<name type="personal">
<namePart type="given">Antoni</namePart>
<namePart type="family">Lasik</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jakub</namePart>
<namePart type="family">Pokrywka</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Łukasz</namePart>
<namePart type="family">Grzybowski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jeremi</namePart>
<namePart type="given">Ignacy</namePart>
<namePart type="family">Kaczmarek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gabriela</namePart>
<namePart type="family">Korzańska</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Janusz</namePart>
<namePart type="family">Świeczkowski-Feiz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Oskar</namePart>
<namePart type="family">Pastuszek</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Paulina</namePart>
<namePart type="family">Hoffman</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jakub</namePart>
<namePart type="given">Tomasz</namePart>
<namePart type="family">Dabrowski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wojciech</namePart>
<namePart type="family">Kusa</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) in medicine are mainly evaluated using multiple-choice question answering (MCQA), which can overestimate real clinical ability due to guessing strategies and answer biases. To address these limitations, we introduce an expanded and more challenging benchmark based on Polish medical exams, adding over 15,000 questions, two new domains, and four structural modifications that reduce MCQA-specific artifacts and better test reasoning. We evaluate 21 LLMs and show that evaluation design strongly affects results. Under our harder setup, the best model (Qwen3.5-122B) drops by 28.4 and 31 pp on English and Polish exams, respectively. Despite low evidence of data contamination, standard MCQA scores do not reliably reflect true medical competence. To facilitate further research, we make our benchmark publicly available: https://github.com/NASK-NLP/Polish-Medical-Exams</abstract>
<identifier type="citekey">lasik-etal-2026-reassessing</identifier>
<identifier type="doi">10.18653/2026.findings-emnlp.2394</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.2394/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>51441</start>
<end>51467</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Reassessing High-Performing LLMs on Polish Medical Exams: True Competence or Bias-Driven Performance?
%A Lasik, Antoni
%A Pokrywka, Jakub
%A Grzybowski, Łukasz
%A Kaczmarek, Jeremi Ignacy
%A Korzańska, Gabriela
%A Świeczkowski-Feiz, Janusz
%A Pastuszek, Oskar
%A Hoffman, Paulina
%A Dabrowski, Jakub Tomasz
%A Kusa, Wojciech
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F lasik-etal-2026-reassessing
%X Large language models (LLMs) in medicine are mainly evaluated using multiple-choice question answering (MCQA), which can overestimate real clinical ability due to guessing strategies and answer biases. To address these limitations, we introduce an expanded and more challenging benchmark based on Polish medical exams, adding over 15,000 questions, two new domains, and four structural modifications that reduce MCQA-specific artifacts and better test reasoning. We evaluate 21 LLMs and show that evaluation design strongly affects results. Under our harder setup, the best model (Qwen3.5-122B) drops by 28.4 and 31 pp on English and Polish exams, respectively. Despite low evidence of data contamination, standard MCQA scores do not reliably reflect true medical competence. To facilitate further research, we make our benchmark publicly available: https://github.com/NASK-NLP/Polish-Medical-Exams
%R 10.18653/2026.findings-emnlp.2394
%U https://aclanthology.org/2026.findings-emnlp.2394/
%U https://doi.org/10.18653/2026.findings-emnlp.2394
%P 51441-51467
Markdown (Informal)
[Reassessing High-Performing LLMs on Polish Medical Exams: True Competence or Bias-Driven Performance?](https://aclanthology.org/2026.findings-emnlp.2394/) (Lasik et al., Findings 2026)
ACL
- Antoni Lasik, Jakub Pokrywka, Łukasz Grzybowski, Jeremi Ignacy Kaczmarek, Gabriela Korzańska, Janusz Świeczkowski-Feiz, Oskar Pastuszek, Paulina Hoffman, Jakub Tomasz Dabrowski, and Wojciech Kusa. 2026. Reassessing High-Performing LLMs on Polish Medical Exams: True Competence or Bias-Driven Performance?. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 51441–51467, Budapest, Hungary. Association for Computational Linguistics.