@inproceedings{christie-etal-2026-fractional,
title = "Fractional Information Gain: A Personalization Metric for Black-Box Predictive Models",
author = "Christie, S. Thomas and
Hauru, Markus and
Rafferty, Anna N.",
editor = "Wilson, Joshua and
Ormerod, Christopher and
Beiting-Parrish, Magdalen",
booktitle = "Proceedings of the Artificial Intelligence in Measurement and Education Conference ({AIME}-Con): Full Papers",
month = oct,
year = "2026",
address = "Wyndham Grand Pittsburgh Downtown, Pittsburgh, Pennsylvania, United States",
publisher = "National Council on Measurement in Education (NCME)",
url = "https://aclanthology.org/2026.aimecon-main.12/",
pages = "109--116",
ISBN = "979-8-9983004-0-0",
abstract = "Black-box knowledge tracing models are commonly deployed to drive personalization in online learning platforms and are typically evaluated using classification metrics such as AUC, accuracy, and F1 score. However, a model that predicts item responses using only each item{'}s proportion correct in the training set achieves AUC up to 0.72 and accuracy up to 0.84 on widely used benchmark datasets, despite using no information about individual students' response histories. Inspired by the concept of marginal reliability in psychometrics, we introduce Fractional Information Gain (FIG), an information-theoretic evaluation metric for black-box predictive models of student item responses. FIG measures the fraction of a student{'}s response uncertainty resolved by a trained model relative to the item-only baseline. FIG equals 0 when the model adds no information beyond item base rates, and 1 when held-out responses are perfectly predicted. FIG is applicable to any model that outputs probabilities and is sensitive to calibration errors. We characterize FIG on synthetic and real data, compare it to AUC for both the item-only baseline and trained models on four benchmark datasets, and describe the operational affordances that FIG inherits from its reliability-like construction. FIG imports the conceptual benefits of score reliability into the prediction-oriented framework of ML-driven educational systems."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="christie-etal-2026-fractional">
<titleInfo>
<title>Fractional Information Gain: A Personalization Metric for Black-Box Predictive Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">S</namePart>
<namePart type="given">Thomas</namePart>
<namePart type="family">Christie</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Markus</namePart>
<namePart type="family">Hauru</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="given">N</namePart>
<namePart type="family">Rafferty</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Artificial Intelligence in Measurement and Education Conference (AIME-Con): Full Papers</title>
</titleInfo>
<name type="personal">
<namePart type="given">Joshua</namePart>
<namePart type="family">Wilson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Christopher</namePart>
<namePart type="family">Ormerod</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Magdalen</namePart>
<namePart type="family">Beiting-Parrish</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>National Council on Measurement in Education (NCME)</publisher>
<place>
<placeTerm type="text">Wyndham Grand Pittsburgh Downtown, Pittsburgh, Pennsylvania, United States</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-9983004-0-0</identifier>
</relatedItem>
<abstract>Black-box knowledge tracing models are commonly deployed to drive personalization in online learning platforms and are typically evaluated using classification metrics such as AUC, accuracy, and F1 score. However, a model that predicts item responses using only each item’s proportion correct in the training set achieves AUC up to 0.72 and accuracy up to 0.84 on widely used benchmark datasets, despite using no information about individual students’ response histories. Inspired by the concept of marginal reliability in psychometrics, we introduce Fractional Information Gain (FIG), an information-theoretic evaluation metric for black-box predictive models of student item responses. FIG measures the fraction of a student’s response uncertainty resolved by a trained model relative to the item-only baseline. FIG equals 0 when the model adds no information beyond item base rates, and 1 when held-out responses are perfectly predicted. FIG is applicable to any model that outputs probabilities and is sensitive to calibration errors. We characterize FIG on synthetic and real data, compare it to AUC for both the item-only baseline and trained models on four benchmark datasets, and describe the operational affordances that FIG inherits from its reliability-like construction. FIG imports the conceptual benefits of score reliability into the prediction-oriented framework of ML-driven educational systems.</abstract>
<identifier type="citekey">christie-etal-2026-fractional</identifier>
<location>
<url>https://aclanthology.org/2026.aimecon-main.12/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>109</start>
<end>116</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Fractional Information Gain: A Personalization Metric for Black-Box Predictive Models
%A Christie, S. Thomas
%A Hauru, Markus
%A Rafferty, Anna N.
%Y Wilson, Joshua
%Y Ormerod, Christopher
%Y Beiting-Parrish, Magdalen
%S Proceedings of the Artificial Intelligence in Measurement and Education Conference (AIME-Con): Full Papers
%D 2026
%8 October
%I National Council on Measurement in Education (NCME)
%C Wyndham Grand Pittsburgh Downtown, Pittsburgh, Pennsylvania, United States
%@ 979-8-9983004-0-0
%F christie-etal-2026-fractional
%X Black-box knowledge tracing models are commonly deployed to drive personalization in online learning platforms and are typically evaluated using classification metrics such as AUC, accuracy, and F1 score. However, a model that predicts item responses using only each item’s proportion correct in the training set achieves AUC up to 0.72 and accuracy up to 0.84 on widely used benchmark datasets, despite using no information about individual students’ response histories. Inspired by the concept of marginal reliability in psychometrics, we introduce Fractional Information Gain (FIG), an information-theoretic evaluation metric for black-box predictive models of student item responses. FIG measures the fraction of a student’s response uncertainty resolved by a trained model relative to the item-only baseline. FIG equals 0 when the model adds no information beyond item base rates, and 1 when held-out responses are perfectly predicted. FIG is applicable to any model that outputs probabilities and is sensitive to calibration errors. We characterize FIG on synthetic and real data, compare it to AUC for both the item-only baseline and trained models on four benchmark datasets, and describe the operational affordances that FIG inherits from its reliability-like construction. FIG imports the conceptual benefits of score reliability into the prediction-oriented framework of ML-driven educational systems.
%U https://aclanthology.org/2026.aimecon-main.12/
%P 109-116
Markdown (Informal)
[Fractional Information Gain: A Personalization Metric for Black-Box Predictive Models](https://aclanthology.org/2026.aimecon-main.12/) (Christie et al., AIME-Con 2026)
ACL