@inproceedings{liu-etal-2026-physdox,
title = "{P}hys{D}ox: Benchmarking {LLM}s on Physical Feasibility Auditing of Physiological {S}ensing Protocols",
author = "Liu, He and
Gu, Boyuan and
Cheng, Shuaiqi and
Sun, Haiyang and
You, Siyu and
Hu, Xuming",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.264/",
pages = "5539--5560",
ISBN = "979-8-89176-462-0",
abstract = "Large language models (LLMs) are increasingly used to assist scientific writing and experimental design, but a fluent protocol can still be physically infeasible. We introduce PhysDox, a benchmark for physical feasibility auditing of biomedical experimental protocols, comprising a 683-sample expert-curated Gold set and a 5,000-sample expanded Silver set across six sensing domains. PhysDox formulates protocol auditing as a two-stage task: severity detection over valid, minor issue, and fatal violation protocols, followed by diagnosis of fatal violations over coarse and fine-grained physical constraints. We evaluate 6 recent LLM variants under 4 inference strategies. Results show that current LLMs remain poorly calibrated at the severity gate: the best Gold Stage-1 macro-F1 is only 53.0, and strong oracle diagnosis often collapses after end-to-end gating. Qualitative case analyses reveal a shared failure mode: models reward complete experimental scaffolds while missing hidden physical constraints. These results show that protocol auditing requires calibrated feasibility reasoning over complete experimental procedures, rather than factual recall or isolated physics problem solving."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2026-physdox">
<titleInfo>
<title>PhysDox: Benchmarking LLMs on Physical Feasibility Auditing of Physiological Sensing Protocols</title>
</titleInfo>
<name type="personal">
<namePart type="given">He</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Boyuan</namePart>
<namePart type="family">Gu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shuaiqi</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Haiyang</namePart>
<namePart type="family">Sun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Siyu</namePart>
<namePart type="family">You</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Xuming</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Large language models (LLMs) are increasingly used to assist scientific writing and experimental design, but a fluent protocol can still be physically infeasible. We introduce PhysDox, a benchmark for physical feasibility auditing of biomedical experimental protocols, comprising a 683-sample expert-curated Gold set and a 5,000-sample expanded Silver set across six sensing domains. PhysDox formulates protocol auditing as a two-stage task: severity detection over valid, minor issue, and fatal violation protocols, followed by diagnosis of fatal violations over coarse and fine-grained physical constraints. We evaluate 6 recent LLM variants under 4 inference strategies. Results show that current LLMs remain poorly calibrated at the severity gate: the best Gold Stage-1 macro-F1 is only 53.0, and strong oracle diagnosis often collapses after end-to-end gating. Qualitative case analyses reveal a shared failure mode: models reward complete experimental scaffolds while missing hidden physical constraints. These results show that protocol auditing requires calibrated feasibility reasoning over complete experimental procedures, rather than factual recall or isolated physics problem solving.</abstract>
<identifier type="citekey">liu-etal-2026-physdox</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.264/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>5539</start>
<end>5560</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PhysDox: Benchmarking LLMs on Physical Feasibility Auditing of Physiological Sensing Protocols
%A Liu, He
%A Gu, Boyuan
%A Cheng, Shuaiqi
%A Sun, Haiyang
%A You, Siyu
%A Hu, Xuming
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F liu-etal-2026-physdox
%X Large language models (LLMs) are increasingly used to assist scientific writing and experimental design, but a fluent protocol can still be physically infeasible. We introduce PhysDox, a benchmark for physical feasibility auditing of biomedical experimental protocols, comprising a 683-sample expert-curated Gold set and a 5,000-sample expanded Silver set across six sensing domains. PhysDox formulates protocol auditing as a two-stage task: severity detection over valid, minor issue, and fatal violation protocols, followed by diagnosis of fatal violations over coarse and fine-grained physical constraints. We evaluate 6 recent LLM variants under 4 inference strategies. Results show that current LLMs remain poorly calibrated at the severity gate: the best Gold Stage-1 macro-F1 is only 53.0, and strong oracle diagnosis often collapses after end-to-end gating. Qualitative case analyses reveal a shared failure mode: models reward complete experimental scaffolds while missing hidden physical constraints. These results show that protocol auditing requires calibrated feasibility reasoning over complete experimental procedures, rather than factual recall or isolated physics problem solving.
%U https://aclanthology.org/2026.findings-emnlp.264/
%P 5539-5560
Markdown (Informal)
[PhysDox: Benchmarking LLMs on Physical Feasibility Auditing of Physiological Sensing Protocols](https://aclanthology.org/2026.findings-emnlp.264/) (Liu et al., Findings 2026)
ACL