@inproceedings{zhang-cheng-2026-whose,
title = "Whose Robots.txt? Consent-Infrastructure Inequality Across the Languages of Web Corpora",
author = "Zhang, Gary and
Cheng, Austin",
editor = "Ortiz Suarez, Pedro and
Burchell, Laurie and
Ljube{\v{s}}i{\'c}, Nikola and
Scherrer, Yves and
Laippala, Veronika and
Dini{\'c}, Vuk and
Vaughan, Thom",
booktitle = "Proceedings of the 13th Web-as-Corpus Workshop",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.wac-1.30/",
pages = "347--355",
ISBN = "979-8-89176-481-1",
abstract = "Website owners increasingly restrict AI crawlers through robots.txt. Audits are English-centric and do not stratify results by language. We measure opt-out for 4,900 domains across fourteen languages and three resource tiers in FineWeb and FineWeb2. Among reachable domains, high-tier sites block 28.8 percent, versus 11.2 percent and 9.8 percent in the mid and low tiers; an exact language-partition permutation gives p = 0.008 for the high-low gap. The gradient is concentrated in naming: files naming any of fifteen AI agents fall from 33.6 percent to 10.6 percent across tiers, while named-agent blocking shows no tier ordering. Among files listing conventional crawlers, AI naming falls from 64.0 percent to 19.0 percent, consistent with vocabulary diffusion rather than maintenance failure. Applying current robots.txt retroactively to an earlier crawl, design-weighted document mass behind a Common Crawl opt-out is 31.2 percent for English; tier means are 26.5 percent, 8.9 percent, and 10.7 percent. Reachability falls from 78 percent in English to 39 percent in Uyghur. These results show a gradient in recorded policy that a consent-respecting rebuild would inherit. We release every fetched robots.txt, derived rates, and code."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-cheng-2026-whose">
<titleInfo>
<title>Whose Robots.txt? Consent-Infrastructure Inequality Across the Languages of Web Corpora</title>
</titleInfo>
<name type="personal">
<namePart type="given">Gary</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Austin</namePart>
<namePart type="family">Cheng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 13th Web-as-Corpus Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Pedro</namePart>
<namePart type="family">Ortiz Suarez</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Laurie</namePart>
<namePart type="family">Burchell</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikola</namePart>
<namePart type="family">Ljubešić</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yves</namePart>
<namePart type="family">Scherrer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Veronika</namePart>
<namePart type="family">Laippala</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Vuk</namePart>
<namePart type="family">Dinić</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Thom</namePart>
<namePart type="family">Vaughan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-481-1</identifier>
</relatedItem>
<abstract>Website owners increasingly restrict AI crawlers through robots.txt. Audits are English-centric and do not stratify results by language. We measure opt-out for 4,900 domains across fourteen languages and three resource tiers in FineWeb and FineWeb2. Among reachable domains, high-tier sites block 28.8 percent, versus 11.2 percent and 9.8 percent in the mid and low tiers; an exact language-partition permutation gives p = 0.008 for the high-low gap. The gradient is concentrated in naming: files naming any of fifteen AI agents fall from 33.6 percent to 10.6 percent across tiers, while named-agent blocking shows no tier ordering. Among files listing conventional crawlers, AI naming falls from 64.0 percent to 19.0 percent, consistent with vocabulary diffusion rather than maintenance failure. Applying current robots.txt retroactively to an earlier crawl, design-weighted document mass behind a Common Crawl opt-out is 31.2 percent for English; tier means are 26.5 percent, 8.9 percent, and 10.7 percent. Reachability falls from 78 percent in English to 39 percent in Uyghur. These results show a gradient in recorded policy that a consent-respecting rebuild would inherit. We release every fetched robots.txt, derived rates, and code.</abstract>
<identifier type="citekey">zhang-cheng-2026-whose</identifier>
<location>
<url>https://aclanthology.org/2026.wac-1.30/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>347</start>
<end>355</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Whose Robots.txt? Consent-Infrastructure Inequality Across the Languages of Web Corpora
%A Zhang, Gary
%A Cheng, Austin
%Y Ortiz Suarez, Pedro
%Y Burchell, Laurie
%Y Ljubešić, Nikola
%Y Scherrer, Yves
%Y Laippala, Veronika
%Y Dinić, Vuk
%Y Vaughan, Thom
%S Proceedings of the 13th Web-as-Corpus Workshop
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-481-1
%F zhang-cheng-2026-whose
%X Website owners increasingly restrict AI crawlers through robots.txt. Audits are English-centric and do not stratify results by language. We measure opt-out for 4,900 domains across fourteen languages and three resource tiers in FineWeb and FineWeb2. Among reachable domains, high-tier sites block 28.8 percent, versus 11.2 percent and 9.8 percent in the mid and low tiers; an exact language-partition permutation gives p = 0.008 for the high-low gap. The gradient is concentrated in naming: files naming any of fifteen AI agents fall from 33.6 percent to 10.6 percent across tiers, while named-agent blocking shows no tier ordering. Among files listing conventional crawlers, AI naming falls from 64.0 percent to 19.0 percent, consistent with vocabulary diffusion rather than maintenance failure. Applying current robots.txt retroactively to an earlier crawl, design-weighted document mass behind a Common Crawl opt-out is 31.2 percent for English; tier means are 26.5 percent, 8.9 percent, and 10.7 percent. Reachability falls from 78 percent in English to 39 percent in Uyghur. These results show a gradient in recorded policy that a consent-respecting rebuild would inherit. We release every fetched robots.txt, derived rates, and code.
%U https://aclanthology.org/2026.wac-1.30/
%P 347-355
Markdown (Informal)
[Whose Robots.txt? Consent-Infrastructure Inequality Across the Languages of Web Corpora](https://aclanthology.org/2026.wac-1.30/) (Zhang & Cheng, WAC 2026)
ACL