@inproceedings{finegan-dollak-2026-helped,
title = "What Helped with Huge Data Hurt with Small Data: Standardization, Representational Geometry, and Data Scale",
author = "Finegan-Dollak, Catherine",
editor = "Choshen, Leshem and
Cotterell, Ryan and
Gul, Mustafa Omer and
Jumelet, Jaap and
Linzen, Tal and
Mueller, Aaron and
Salhan, Suchir and
Shah, Raj Sanjay and
Warstadt, Alex and
Wilcox, Ethan Gotlieb",
booktitle = "Proceedings of the Second {B}aby{LM} Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.babylm-main.31/",
pages = "388--402",
ISBN = "979-8-89176-475-0",
abstract = "Training models on cognitively plausible (CP) amounts of data (tens or hundreds of millions of words) offers multiple advantages over the huge-data training (hundreds of billions to trillions of tokens) typical of current models, but achieving competitive performance at CP scale remains an open challenge. Understanding differences in representational geometry between CP- and huge-data models may help us design better CP-scale models. We identify differences in two aspects of representational geometry (anisotropy and rogue dimensionality) and a difference in effect of a geometry-targeted intervention.\par Although standardization{---}a technique to address anisotropy and rogue dimensions in LLM representational geometry{---}improves lexical similarity performance in huge-data models, the effect reverses for CP-data models. We also find that both rogue dimensionality and anisotropy correlate with data scale, further supporting the view that CP- and huge-scale models differ geometrically. However, statistical analysis of results across 18 models showed that neither rogue dimensions nor anisotropy significantly positively mediated the difference in standardization effect, except in one model per mechanism. This absence of mediation is itself a finding: it fails to support the two mechanisms most directly motivated by prior work on standardization, narrowing what plausibly explains the reversal. Thus, the observed qualitative difference in standardization effect may hint at as-yet undiscovered differences, geometric or otherwise, between CP-scale models and models trained on huge data."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="finegan-dollak-2026-helped">
<titleInfo>
<title>What Helped with Huge Data Hurt with Small Data: Standardization, Representational Geometry, and Data Scale</title>
</titleInfo>
<name type="personal">
<namePart type="given">Catherine</namePart>
<namePart type="family">Finegan-Dollak</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Second BabyLM Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leshem</namePart>
<namePart type="family">Choshen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ryan</namePart>
<namePart type="family">Cotterell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mustafa</namePart>
<namePart type="given">Omer</namePart>
<namePart type="family">Gul</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jaap</namePart>
<namePart type="family">Jumelet</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tal</namePart>
<namePart type="family">Linzen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Aaron</namePart>
<namePart type="family">Mueller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Suchir</namePart>
<namePart type="family">Salhan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Raj</namePart>
<namePart type="given">Sanjay</namePart>
<namePart type="family">Shah</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alex</namePart>
<namePart type="family">Warstadt</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ethan</namePart>
<namePart type="given">Gotlieb</namePart>
<namePart type="family">Wilcox</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-475-0</identifier>
</relatedItem>
<abstract>Training models on cognitively plausible (CP) amounts of data (tens or hundreds of millions of words) offers multiple advantages over the huge-data training (hundreds of billions to trillions of tokens) typical of current models, but achieving competitive performance at CP scale remains an open challenge. Understanding differences in representational geometry between CP- and huge-data models may help us design better CP-scale models. We identify differences in two aspects of representational geometry (anisotropy and rogue dimensionality) and a difference in effect of a geometry-targeted intervention.\par Although standardization—a technique to address anisotropy and rogue dimensions in LLM representational geometry—improves lexical similarity performance in huge-data models, the effect reverses for CP-data models. We also find that both rogue dimensionality and anisotropy correlate with data scale, further supporting the view that CP- and huge-scale models differ geometrically. However, statistical analysis of results across 18 models showed that neither rogue dimensions nor anisotropy significantly positively mediated the difference in standardization effect, except in one model per mechanism. This absence of mediation is itself a finding: it fails to support the two mechanisms most directly motivated by prior work on standardization, narrowing what plausibly explains the reversal. Thus, the observed qualitative difference in standardization effect may hint at as-yet undiscovered differences, geometric or otherwise, between CP-scale models and models trained on huge data.</abstract>
<identifier type="citekey">finegan-dollak-2026-helped</identifier>
<location>
<url>https://aclanthology.org/2026.babylm-main.31/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>388</start>
<end>402</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T What Helped with Huge Data Hurt with Small Data: Standardization, Representational Geometry, and Data Scale
%A Finegan-Dollak, Catherine
%Y Choshen, Leshem
%Y Cotterell, Ryan
%Y Gul, Mustafa Omer
%Y Jumelet, Jaap
%Y Linzen, Tal
%Y Mueller, Aaron
%Y Salhan, Suchir
%Y Shah, Raj Sanjay
%Y Warstadt, Alex
%Y Wilcox, Ethan Gotlieb
%S Proceedings of the Second BabyLM Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-475-0
%F finegan-dollak-2026-helped
%X Training models on cognitively plausible (CP) amounts of data (tens or hundreds of millions of words) offers multiple advantages over the huge-data training (hundreds of billions to trillions of tokens) typical of current models, but achieving competitive performance at CP scale remains an open challenge. Understanding differences in representational geometry between CP- and huge-data models may help us design better CP-scale models. We identify differences in two aspects of representational geometry (anisotropy and rogue dimensionality) and a difference in effect of a geometry-targeted intervention.\par Although standardization—a technique to address anisotropy and rogue dimensions in LLM representational geometry—improves lexical similarity performance in huge-data models, the effect reverses for CP-data models. We also find that both rogue dimensionality and anisotropy correlate with data scale, further supporting the view that CP- and huge-scale models differ geometrically. However, statistical analysis of results across 18 models showed that neither rogue dimensions nor anisotropy significantly positively mediated the difference in standardization effect, except in one model per mechanism. This absence of mediation is itself a finding: it fails to support the two mechanisms most directly motivated by prior work on standardization, narrowing what plausibly explains the reversal. Thus, the observed qualitative difference in standardization effect may hint at as-yet undiscovered differences, geometric or otherwise, between CP-scale models and models trained on huge data.
%U https://aclanthology.org/2026.babylm-main.31/
%P 388-402
Markdown (Informal)
[What Helped with Huge Data Hurt with Small Data: Standardization, Representational Geometry, and Data Scale](https://aclanthology.org/2026.babylm-main.31/) (Finegan-Dollak, BabyLM 2026)
ACL