@inproceedings{amun-etal-2026-psychometric,
title = "Psychometric Benchmark and Bias Assessment in Open-Source Pre-trained {NLI} Models",
author = "Amun, Fadi and
Cohen, Ofir and
Elyashar, Aviad and
Cohen, Odeya and
Puzis, Rami",
editor = "Zhang, Yue and
Xu, Ruifeng and
Purwarianti, Ayu and
Rong, Tong and
Suzuki, Jun",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {IJCNLP}-{AACL} 2026",
month = nov,
year = "2026",
address = "Hengqin, China",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-aacl.189/",
pages = "3753--3771",
ISBN = "979-8-89176-444-6",
abstract = "Pre-trained language models harbor latent psychological biases inherited from training data, which remain largely unmeasured, creating risks in areas ranging from mental health support to professional decision making. In this study, the evaluation of model psychometrics and assessment of latent constructs framework is employed to evaluate psychometric biases in 334 Hugging Face natural language inference models, assessing the models' anxiety, depression, sense of coherence, sexism, compassion, and personality. Models consistently score above zero on positively valenced constructs (e.g., compassion, sense of coherence) and below zero on negatively valenced constructs (e.g., anxiety, depression), favoring positive over negative psychological language. Architecture and size are associated with systematic differences: DeBERTa models show elevated negatively valenced constructs (e.g., PHQ {\ensuremath{\delta}} = +0.45, GAD {\ensuremath{\delta}} = +0.40) and reduced positively valenced constructs (e.g., Extraversion {\ensuremath{\delta}} = {\ensuremath{-}}0.61), while small models show the opposite pattern (e.g., Extraversion {\ensuremath{\delta}} = +0.30, PHQ {\ensuremath{\delta}} = {\ensuremath{-}}0.22), with both effects surviving multivariate control. Among the 105 models reporting training data, fine-tuning on adversarial natural language inference was associated with elevated depression ({\ensuremath{\delta}} = +0.65), while year of release showed no significant effects after multivariate control. These results suggest that psychometric profiling is a critical, yet missing, layer of safety auditing for NLI models."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="amun-etal-2026-psychometric">
<titleInfo>
<title>Psychometric Benchmark and Bias Assessment in Open-Source Pre-trained NLI Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Fadi</namePart>
<namePart type="family">Amun</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ofir</namePart>
<namePart type="family">Cohen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aviad</namePart>
<namePart type="family">Elyashar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Odeya</namePart>
<namePart type="family">Cohen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Rami</namePart>
<namePart type="family">Puzis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yue</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruifeng</namePart>
<namePart type="family">Xu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ayu</namePart>
<namePart type="family">Purwarianti</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Rong</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jun</namePart>
<namePart type="family">Suzuki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Hengqin, China</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-444-6</identifier>
</relatedItem>
<abstract>Pre-trained language models harbor latent psychological biases inherited from training data, which remain largely unmeasured, creating risks in areas ranging from mental health support to professional decision making. In this study, the evaluation of model psychometrics and assessment of latent constructs framework is employed to evaluate psychometric biases in 334 Hugging Face natural language inference models, assessing the models’ anxiety, depression, sense of coherence, sexism, compassion, and personality. Models consistently score above zero on positively valenced constructs (e.g., compassion, sense of coherence) and below zero on negatively valenced constructs (e.g., anxiety, depression), favoring positive over negative psychological language. Architecture and size are associated with systematic differences: DeBERTa models show elevated negatively valenced constructs (e.g., PHQ \ensuremathδ = +0.45, GAD \ensuremathδ = +0.40) and reduced positively valenced constructs (e.g., Extraversion \ensuremathδ = \ensuremath-0.61), while small models show the opposite pattern (e.g., Extraversion \ensuremathδ = +0.30, PHQ \ensuremathδ = \ensuremath-0.22), with both effects surviving multivariate control. Among the 105 models reporting training data, fine-tuning on adversarial natural language inference was associated with elevated depression (\ensuremathδ = +0.65), while year of release showed no significant effects after multivariate control. These results suggest that psychometric profiling is a critical, yet missing, layer of safety auditing for NLI models.</abstract>
<identifier type="citekey">amun-etal-2026-psychometric</identifier>
<location>
<url>https://aclanthology.org/2026.findings-aacl.189/</url>
</location>
<part>
<date>2026-11</date>
<extent unit="page">
<start>3753</start>
<end>3771</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Psychometric Benchmark and Bias Assessment in Open-Source Pre-trained NLI Models
%A Amun, Fadi
%A Cohen, Ofir
%A Elyashar, Aviad
%A Cohen, Odeya
%A Puzis, Rami
%Y Zhang, Yue
%Y Xu, Ruifeng
%Y Purwarianti, Ayu
%Y Rong, Tong
%Y Suzuki, Jun
%S Findings of the Association for Computational Linguistics: IJCNLP-AACL 2026
%D 2026
%8 November
%I Association for Computational Linguistics
%C Hengqin, China
%@ 979-8-89176-444-6
%F amun-etal-2026-psychometric
%X Pre-trained language models harbor latent psychological biases inherited from training data, which remain largely unmeasured, creating risks in areas ranging from mental health support to professional decision making. In this study, the evaluation of model psychometrics and assessment of latent constructs framework is employed to evaluate psychometric biases in 334 Hugging Face natural language inference models, assessing the models’ anxiety, depression, sense of coherence, sexism, compassion, and personality. Models consistently score above zero on positively valenced constructs (e.g., compassion, sense of coherence) and below zero on negatively valenced constructs (e.g., anxiety, depression), favoring positive over negative psychological language. Architecture and size are associated with systematic differences: DeBERTa models show elevated negatively valenced constructs (e.g., PHQ \ensuremathδ = +0.45, GAD \ensuremathδ = +0.40) and reduced positively valenced constructs (e.g., Extraversion \ensuremathδ = \ensuremath-0.61), while small models show the opposite pattern (e.g., Extraversion \ensuremathδ = +0.30, PHQ \ensuremathδ = \ensuremath-0.22), with both effects surviving multivariate control. Among the 105 models reporting training data, fine-tuning on adversarial natural language inference was associated with elevated depression (\ensuremathδ = +0.65), while year of release showed no significant effects after multivariate control. These results suggest that psychometric profiling is a critical, yet missing, layer of safety auditing for NLI models.
%U https://aclanthology.org/2026.findings-aacl.189/
%P 3753-3771
Markdown (Informal)
[Psychometric Benchmark and Bias Assessment in Open-Source Pre-trained NLI Models](https://aclanthology.org/2026.findings-aacl.189/) (Amun et al., Findings 2026)
ACL