@inproceedings{upadhyay-etal-2026-praxis,
title = "{PRAXIS}: A 68-Industry Benchmark for Pragmatic-Form Bias in Suggestion Mining",
author = "Upadhyay, Aniket and
Trivedi, Aakash and
Bajoria, Sparsh and
Ranjan, Shreyanshu and
Kumar, Dhruv and
Narang, Pratik",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.741/",
pages = "15566--15586",
ISBN = "979-8-89176-462-0",
abstract = "Suggestion mining has traditionally been formulated as binary classification, but the underlying linguistic phenomenon is not binary. A single review can express the same operational recommendation through a direct imperative (e.g., ``add a vegetarian entr{\'e}e''), an optative wish, an interrogative request, or a hedged conditional. These utterances share illocutionary force but differ substantially in pragmatic form, placing different demands on models: lexical cues may suffice for direct suggestions, whereas indirect forms require pragmatic interpretation. Existing benchmarks, typically spanning one to five domains and using a single binary label, do not measure these distinctions.\par We introduce PRAXIS, an English consumer-review benchmark that reframes suggestion mining as the measurement of pragmatic-form bias rather than as a leaderboard task. PRAXIS provides (i) a silver corpus of 174,860 reviews spanning 68 industries, labeled by Claude Opus 4.7; (ii) a 15,000-example human-labeled gold evaluation set, with annotation reliability estimated by a three-annotator panel on a 1,000-example subset (Tier 1 Fleiss' {\ensuremath{\kappa}} = 0.77); and (iii) evaluations of four model families against the human gold, with bootstrap 95{\%} confidence intervals. Agreement between Opus and human labels is reported separately as a corpus-validation statistic (Tier 1 Cohen{'}s {\ensuremath{\kappa}} = 0.71), rather than as inter-annotator agreement.\par Our central finding is structural rather than competitive. Instruction-tuned LLMs outperform encoders on binary macro-F1 (DeepSeek-V4-Pro: 0.882 vs. RoBERTa-large: 0.780), yet the ranking reverses for the Tier 1 conditional form: RoBERTa-large recalls 85.2{\%} of conditional suggestions, compared with 46.7{\%} for the strongest zero-shot LLM and 8.0{--}43.1{\%} for the other six LLMs. This discrepancy reflects a form-classification gap rather than a detection gap: models often recognize these instances as suggestions at the binary level but misclassify their pragmatic form. A matched-supervision QLoRA model based on Qwen2.5-7B, trained on the same silver labels used for the encoders, recovers encoder-level conditional recall (0.904). PRAXIS therefore exposes a structural error mode that flat binary metrics obscure, positioning the benchmark as a measurement instrument for pragmatic-form bias rather than a conventional model leaderboard."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="upadhyay-etal-2026-praxis">
<titleInfo>
<title>PRAXIS: A 68-Industry Benchmark for Pragmatic-Form Bias in Suggestion Mining</title>
</titleInfo>
<name type="personal">
<namePart type="given">Aniket</namePart>
<namePart type="family">Upadhyay</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aakash</namePart>
<namePart type="family">Trivedi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sparsh</namePart>
<namePart type="family">Bajoria</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Shreyanshu</namePart>
<namePart type="family">Ranjan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dhruv</namePart>
<namePart type="family">Kumar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pratik</namePart>
<namePart type="family">Narang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Suggestion mining has traditionally been formulated as binary classification, but the underlying linguistic phenomenon is not binary. A single review can express the same operational recommendation through a direct imperative (e.g., “add a vegetarian entrée”), an optative wish, an interrogative request, or a hedged conditional. These utterances share illocutionary force but differ substantially in pragmatic form, placing different demands on models: lexical cues may suffice for direct suggestions, whereas indirect forms require pragmatic interpretation. Existing benchmarks, typically spanning one to five domains and using a single binary label, do not measure these distinctions.\par We introduce PRAXIS, an English consumer-review benchmark that reframes suggestion mining as the measurement of pragmatic-form bias rather than as a leaderboard task. PRAXIS provides (i) a silver corpus of 174,860 reviews spanning 68 industries, labeled by Claude Opus 4.7; (ii) a 15,000-example human-labeled gold evaluation set, with annotation reliability estimated by a three-annotator panel on a 1,000-example subset (Tier 1 Fleiss’ \ensuremathąppa = 0.77); and (iii) evaluations of four model families against the human gold, with bootstrap 95% confidence intervals. Agreement between Opus and human labels is reported separately as a corpus-validation statistic (Tier 1 Cohen’s \ensuremathąppa = 0.71), rather than as inter-annotator agreement.\par Our central finding is structural rather than competitive. Instruction-tuned LLMs outperform encoders on binary macro-F1 (DeepSeek-V4-Pro: 0.882 vs. RoBERTa-large: 0.780), yet the ranking reverses for the Tier 1 conditional form: RoBERTa-large recalls 85.2% of conditional suggestions, compared with 46.7% for the strongest zero-shot LLM and 8.0–43.1% for the other six LLMs. This discrepancy reflects a form-classification gap rather than a detection gap: models often recognize these instances as suggestions at the binary level but misclassify their pragmatic form. A matched-supervision QLoRA model based on Qwen2.5-7B, trained on the same silver labels used for the encoders, recovers encoder-level conditional recall (0.904). PRAXIS therefore exposes a structural error mode that flat binary metrics obscure, positioning the benchmark as a measurement instrument for pragmatic-form bias rather than a conventional model leaderboard.</abstract>
<identifier type="citekey">upadhyay-etal-2026-praxis</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.741/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>15566</start>
<end>15586</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T PRAXIS: A 68-Industry Benchmark for Pragmatic-Form Bias in Suggestion Mining
%A Upadhyay, Aniket
%A Trivedi, Aakash
%A Bajoria, Sparsh
%A Ranjan, Shreyanshu
%A Kumar, Dhruv
%A Narang, Pratik
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F upadhyay-etal-2026-praxis
%X Suggestion mining has traditionally been formulated as binary classification, but the underlying linguistic phenomenon is not binary. A single review can express the same operational recommendation through a direct imperative (e.g., “add a vegetarian entrée”), an optative wish, an interrogative request, or a hedged conditional. These utterances share illocutionary force but differ substantially in pragmatic form, placing different demands on models: lexical cues may suffice for direct suggestions, whereas indirect forms require pragmatic interpretation. Existing benchmarks, typically spanning one to five domains and using a single binary label, do not measure these distinctions.\par We introduce PRAXIS, an English consumer-review benchmark that reframes suggestion mining as the measurement of pragmatic-form bias rather than as a leaderboard task. PRAXIS provides (i) a silver corpus of 174,860 reviews spanning 68 industries, labeled by Claude Opus 4.7; (ii) a 15,000-example human-labeled gold evaluation set, with annotation reliability estimated by a three-annotator panel on a 1,000-example subset (Tier 1 Fleiss’ \ensuremathąppa = 0.77); and (iii) evaluations of four model families against the human gold, with bootstrap 95% confidence intervals. Agreement between Opus and human labels is reported separately as a corpus-validation statistic (Tier 1 Cohen’s \ensuremathąppa = 0.71), rather than as inter-annotator agreement.\par Our central finding is structural rather than competitive. Instruction-tuned LLMs outperform encoders on binary macro-F1 (DeepSeek-V4-Pro: 0.882 vs. RoBERTa-large: 0.780), yet the ranking reverses for the Tier 1 conditional form: RoBERTa-large recalls 85.2% of conditional suggestions, compared with 46.7% for the strongest zero-shot LLM and 8.0–43.1% for the other six LLMs. This discrepancy reflects a form-classification gap rather than a detection gap: models often recognize these instances as suggestions at the binary level but misclassify their pragmatic form. A matched-supervision QLoRA model based on Qwen2.5-7B, trained on the same silver labels used for the encoders, recovers encoder-level conditional recall (0.904). PRAXIS therefore exposes a structural error mode that flat binary metrics obscure, positioning the benchmark as a measurement instrument for pragmatic-form bias rather than a conventional model leaderboard.
%U https://aclanthology.org/2026.findings-emnlp.741/
%P 15566-15586
Markdown (Informal)
[PRAXIS: A 68-Industry Benchmark for Pragmatic-Form Bias in Suggestion Mining](https://aclanthology.org/2026.findings-emnlp.741/) (Upadhyay et al., Findings 2026)
ACL