@inproceedings{laurito-etal-2024-cluster,
title = "Cluster-Norm for Unsupervised Probing of Knowledge",
author = {Laurito, Walter and
Maiya, Sharan and
Dhimo{\"\i}la, Gr{\'e}goire and
Yeung, Owen and
H{\"a}nni, Kaarel},
editor = "Al-Onaizan, Yaser and
Bansal, Mohit and
Chen, Yun-Nung",
booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing",
month = nov,
year = "2024",
address = "Miami, Florida, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.emnlp-main.780",
pages = "14083--14112",
abstract = "The deployment of language models brings challenges in generating reliable text, especially when these models are fine-tuned with human preferences. To extract the encoded knowledge in these models without (potentially) biased human labels, unsupervised probing techniques like Contrast-Consistent Search (CCS) have been developed (Burns et al., 2022). However, salient but unrelated features in activation space can mislead these probes (Farquhar et al., 2023). Addressing this, we propose a cluster-normalization method to minimize the impact of such features by clustering and normalizing activations of contrast pairs before applying unsupervised probing techniques. While this approach does not address the issue of distinguishing between latent knowledge and that portrayed by a simulated agent{---}a major issue in the literature of eliciting latent knowledge (Paul Christiano and Xu, 2021){---}it still significantly improves the accuracy of probes in identifying the intended knowledge amidst distractions.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="laurito-etal-2024-cluster">
<titleInfo>
<title>Cluster-Norm for Unsupervised Probing of Knowledge</title>
</titleInfo>
<name type="personal">
<namePart type="given">Walter</namePart>
<namePart type="family">Laurito</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sharan</namePart>
<namePart type="family">Maiya</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Grégoire</namePart>
<namePart type="family">Dhimoïla</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Owen</namePart>
<namePart type="family">Yeung</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kaarel</namePart>
<namePart type="family">Hänni</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yaser</namePart>
<namePart type="family">Al-Onaizan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohit</namePart>
<namePart type="family">Bansal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yun-Nung</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Miami, Florida, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>The deployment of language models brings challenges in generating reliable text, especially when these models are fine-tuned with human preferences. To extract the encoded knowledge in these models without (potentially) biased human labels, unsupervised probing techniques like Contrast-Consistent Search (CCS) have been developed (Burns et al., 2022). However, salient but unrelated features in activation space can mislead these probes (Farquhar et al., 2023). Addressing this, we propose a cluster-normalization method to minimize the impact of such features by clustering and normalizing activations of contrast pairs before applying unsupervised probing techniques. While this approach does not address the issue of distinguishing between latent knowledge and that portrayed by a simulated agent—a major issue in the literature of eliciting latent knowledge (Paul Christiano and Xu, 2021)—it still significantly improves the accuracy of probes in identifying the intended knowledge amidst distractions.</abstract>
<identifier type="citekey">laurito-etal-2024-cluster</identifier>
<location>
<url>https://aclanthology.org/2024.emnlp-main.780</url>
</location>
<part>
<date>2024-11</date>
<extent unit="page">
<start>14083</start>
<end>14112</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Cluster-Norm for Unsupervised Probing of Knowledge
%A Laurito, Walter
%A Maiya, Sharan
%A Dhimoïla, Grégoire
%A Yeung, Owen
%A Hänni, Kaarel
%Y Al-Onaizan, Yaser
%Y Bansal, Mohit
%Y Chen, Yun-Nung
%S Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing
%D 2024
%8 November
%I Association for Computational Linguistics
%C Miami, Florida, USA
%F laurito-etal-2024-cluster
%X The deployment of language models brings challenges in generating reliable text, especially when these models are fine-tuned with human preferences. To extract the encoded knowledge in these models without (potentially) biased human labels, unsupervised probing techniques like Contrast-Consistent Search (CCS) have been developed (Burns et al., 2022). However, salient but unrelated features in activation space can mislead these probes (Farquhar et al., 2023). Addressing this, we propose a cluster-normalization method to minimize the impact of such features by clustering and normalizing activations of contrast pairs before applying unsupervised probing techniques. While this approach does not address the issue of distinguishing between latent knowledge and that portrayed by a simulated agent—a major issue in the literature of eliciting latent knowledge (Paul Christiano and Xu, 2021)—it still significantly improves the accuracy of probes in identifying the intended knowledge amidst distractions.
%U https://aclanthology.org/2024.emnlp-main.780
%P 14083-14112
Markdown (Informal)
[Cluster-Norm for Unsupervised Probing of Knowledge](https://aclanthology.org/2024.emnlp-main.780) (Laurito et al., EMNLP 2024)
ACL
- Walter Laurito, Sharan Maiya, Grégoire Dhimoïla, Owen Yeung, and Kaarel Hänni. 2024. Cluster-Norm for Unsupervised Probing of Knowledge. In Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pages 14083–14112, Miami, Florida, USA. Association for Computational Linguistics.