% ahis must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.

\pdfoutput=1


% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
% \usepackage[]{acl}
\usepackage{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{graphicx}
\usepackage{amsmath}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
\usepackage{tikz}

\definecolor{beige}{HTML}{FFF6E0}
\definecolor{darkgray}{HTML}{1A1A1A}
\definecolor{bordergray}{HTML}{4D4A43}

\newcommand{\circlednumber}[1]{%
  \begin{tikzpicture}[baseline=-0.8ex, scale=0.8, transform shape]
    \node[shape=circle, draw=bordergray, line width=1pt, inner sep=0.1ex, fill=beige, text=darkgray, yshift=-0.3ex, xshift=-0.1ex,text width=1em, align=center] (char) {\sffamily\scriptsize\textbf{#1}};
  \end{tikzpicture}
  \kern-0.3em
}


% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Moral Mimicry: Large Language Models Produce Moral Rationalizations
Tailored to Political Identity}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Gabriel Simmons \\
  UC Davis\\
  \texttt{gsimmons@ucdavis.edu}}

\begin{document}

\setlength{\abovedisplayskip}{4pt}
\setlength{\belowdisplayskip}{4pt}

\maketitle
\begin{abstract}
Large Language Models (LLMs) have demonstrated impressive capabilities in generating fluent text, as well as tendencies to reproduce undesirable social biases. This study investigates whether LLMs reproduce the moral biases associated with political groups in the United States, an instance of a broader capability herein termed \textit{moral mimicry}. This hypothesis is explored in the GPT-3/3.5 and OPT families of Transformer-based LLMs. Using tools from Moral Foundations Theory, it is shown that these LLMs are indeed moral mimics. When prompted with a liberal or conservative political identity, the models generate text reflecting corresponding moral biases. This study also explores the relationship between moral mimicry and model size, and similarity between human and LLM moral word use. 
\end{abstract}

\section{Introduction}

\begin{figure*}
\centering
\includegraphics[width=1\textwidth]{figures/Fig1-Horizontal-v2.png}

\caption{An example of the experimental methods. Prompts \circlednumber{2} are constructed from scenarios \circlednumber{1a},  identity phrases \circlednumber{1b}, and stances \circlednumber{1c}, combined in a template (Section \ref{sec:prompt_construction}). Text completions \circlednumber{3} are generated by LLMs based on the prompts (Section \ref{sec:text_generation}). The completions are analyzed for their foundational contents \circlednumber{4} using the moral foundations dictionaries (Section \ref{sec:text_evaluation}). Differences between texts generated from liberal and conservative prompting are used to calculate effect sizes \circlednumber{5}. }\label{fig:fig1}
\end{figure*}


Recent work suggests that Large Language Model (LLM) performance will continue to scale with model and training data sizes
\cite{kaplanScalingLawsNeural2020}. As LLMs
advance in capability, it becomes more likely that they will be capable
of producing text that influences human opinions
\cite{tikuGoogleEngineerWho2022}, potentially lowering
barriers to disinformation
\cite{weidingerTaxonomyRisksPosed2022}. More
optimistically, LLMs may play a role in bridging
divides between social groups
\cite{alshomaryAudienceawareArgumentGeneration2021,jiangCommunityLMProbingPartisan2022}.
For better or worse, we should understand how LLM-generated content will
impact the human informational environment - whether this content is
influential, and to whom.

Morality is an important factor in persuasiveness and polarization of
human opinions
\cite{luttrellChallengingMoralAttitudes2019}.
Moral argumentation can modulate willingness to compromise
\cite{kodapanakkalMoralFramesAre2022}, and
moral congruence between participants in a dialogue influences argument
effectiveness
\cite{feinbergGulfBridgeWhen2015} and
perceptions of ethicality
\cite{egorovItMatchMoralization2020}.

Therefore, it is important to characterize the capabilities of LLMs to produce apparently-moral content\footnote{Anthropomorphization provides convenient ways to talk about system behavior, but can also distort perception of underlying mechanisms \cite{benderClimbingNLUMeaning2020}. To be clear, I ascribe capabilities such as ``moral argumentation'' or ``moral congruence'' to language models only to the extent that their outputs may be perceived as such, and make no claim that LLMs might generate such text with communicative intent.}. This requires a framework from
which we can study morality; Moral Foundations Theory (MFT) is one such framework. MFT proposes that human morals rely on five foundations: Care/Harm, Fairness/Cheating, Loyalty/Betrayal, Authority/Subversion, and Sanctity/Degradation\footnote{Liberty/Oppression was proposed as a sixth foundation - for the sake of this analysis I consider only the original 5 foundations, as these are the ones available in the Moral Foundations Dictionaries \cite{grahamLiberalsConservativesRely2009,frimerMoralFoundationsDictionary2019,hoppExtendedMoralFoundations2021}.}.
Evidence from MFT supports the ``Moral Foundations Hypothesis'' that political groups in the United States vary in their foundation use -
liberals rely primarily on the individualizing foundations (Care/Harm and Fairness/Cheating), while conservatives make more balanced appeals
to all 5 foundations, appealing to the binding foundations (Authority/Subversion, Sanctity/Degradation, and Loyalty/Betrayal) more than liberals \cite{grahamLiberalsConservativesRely2009,dogruyolFivefactorModelMoral2019,frimerLiberalsConservativesUse2020}.

Existing work has investigated the moral foundational biases of language
models that have been fine-tuned on supervised data
\cite{fraserDoesMoralCode2022}, investigated whether language models reproduce other social biases (see \cite{weidingerTaxonomyRisksPosed2022} section 2.1.1), and probed LLMs for differences in other cultural values \cite{aroraProbingPreTrainedLanguage2022}. Concurrent work has shown that LLMs used as dialog agents tend to repeat users' political views back to them, and that this happens more
frequently in larger models \cite{perezDiscoveringLanguageModel2022}. To my knowledge, no work yet examines whether language models can perform \emph{moral mimicry} - that is, reproduce the moral foundational biases associated with social groups such as political identities.

The present study considers whether LLMs use moral vocabulary in ways that are situationally-appropriate, and how this compares to human foundation use. I find that LLMs respond to the salient moral attributes of scenario descriptions, increasing their use of the appropriate foundations, but still differ from human consensus foundation use more than individual humans (Section \ref{sec:methods_results_2}). I then turn to the moral mimicry phenomenon. I investigate whether conditioning an LLM with a political ``identity'' influences the model's use of moral foundations in ways that are consistent with human moral biases. I find confirmatory results for text generated based on``liberal'' and ``conservative'' political identities (Section \ref{sec:methods_results_3}). Finally, I ask how the moral mimicry phenomenon varies with model size. Results show that the extent to which LLMs can reproduce moral biases increases with model size, in the OPT family (Section \ref{sec:methods_results_3}). This is also true for the GPT-3 and -3.5 models considered together, and to a lesser extent for the GPT-3 models alone. 

\hypertarget{methods}{%
\section{Methods}\label{methods}}

\hypertarget{data-generation}{%
\paragraph{Data Generation}\label{data-generation}}

All experiments follow the same pattern for data generation, described in the following sections and illustrated in Figure \ref{fig:fig1}. Methods accompanying specific research questions are presented alongside results in Sections \ref{sec:methods_results_2} - \ref{sec:methods_results_6}. 
\hypertarget{sec:prompt_construction}{%
\paragraph{Prompt Construction}\label{sec:prompt_construction}}

I constructed prompts that encourage the language model to generate apparent moral rationalizations. Each prompt conditions the model with three variables: a scenario \(s\), a political identity phrase \(i\), and a moral stance \(r\). Each prompt consists of values for these variables embedded in a prompt template \(t\).

\textbf{Scenarios} are text strings describing situations or actions apt for moral judgement. I used three datasets (Moral Stories\footnote{Downloaded from https://github.com/demelin/moral\_stories} \cite{emelinMoralStoriesSituated2021}, ETHICS\footnote{Downloaded from https://github.com/hendrycks/ethics} \cite{hendrycksAligningAIShared2021}, and Social Chemistry 101\footnote{Downloaded from https://github.com/mbforbes/social-chemistry-101} \cite{forbesSocialChemistry1012021}) to
obtain four sets of scenarios, which I refer to as Moral Stories, ETHICS, Social Chemistry Actions, and Social Chemistry Situations. Appendix Section \ref{sec:additional_dataset_information} provides specifics on how each dataset was constructed. I use \(S\) and \(s\) to a set of scenarios, and a single scenario, respectively.

\textbf{Political identity phrases} are text strings referring to political ideologies (e.g.~``liberal''). I use \(I\) and \(i\) to refer
to a set of political identities and an individual identity, respectively.

\textbf{Moral Stances} The moral stance presented in each prompt conditions the model to produce an apparent rationalization indicating approval or disapproval of the scenario. I use \(R, r\) to refer to the set of stances \(\{\text{moral}, \text{immoral}\}\), and a single stance, respectively. The datasets used herein contain labels indicating the normative moral acceptability of each scenario. For a scenario \(s\), I refer to its normative moral acceptability as \(r_H(s)\). 

\textbf{Prompt Templates} are functions that convert a tuple of scenario, identity phrase, and moral stance into a prompt. To check for sensitivity to any particular phrasing, five different styles of prompt template were used (see Appendix Tables \ref{tbl:prompt_styles_situations}
and \ref{tbl:prompt_styles_actions}). Prompts were constructed by selecting a template \(t\) for a particular style, and populating it with a stance, scenario, and political identity phrase.  

\hypertarget{sec:text_generation}{%
\paragraph{Text Generation with LLMs}\label{sec:text_generation}}

Language models produce text by autoregressive decoding. Given a sequence of tokens, the model assigns likelihoods to all tokens in its vocabulary indicating how likely they are to follow the sequence. Based on these likelihoods, a suitable next token is appended to the sequence, and the process is repeated until a maximum number of tokens is generated, or the model generates a special ``end-of-sequence'' token. I refer to the text provided initially to the model as a ``prompt'' and the text obtained through the decoding process as a ``completion''. In this work I used three families of Large Language Models: GPT-3, GPT-3.5, and OPT (Table \ref{tbl:model_details}). GPT-3 is a family of Transformer-based
\cite{vaswaniAttentionAllYou2017}
autoregressive language models with sizes up to 175 billion parameters, pre-trained in self-supervised fashion on web text corpora
\cite{radfordLanguageModelsAre}. The largest 3 of the 4 GPT-3 models evaluated here also received supervised fine-tuning on high-quality model samples and human demonstrations \cite{openaiModelIndexResearchers}. The GPT-3.5 models are also Transformer-based, pre-trained on text and code web corpora, and fine-tuned using either supervised fine-tuning or reinforcement learning from human preferences \cite{openaiModelIndexResearchers}. I accessed GPT-3/3.5 through the OpenAI Completions API \cite{OpenAIAPI2021}. I used the engine parameter to indicate a specific model. GPT-3 models ``text-ada-001'', ``text-babbage-001'', ``text-curie-001'', and ``text-davinci-001'', and GPT-3.5 models ``text-davinci-002'' and ``text-davinci-003'' were used. The OPT models are Transformer-based pre-trained models released by Meta AI, with sizes up to 175B parameters \cite{zhangOPTOpenPretrained2022}. Model sizes up to 30B parameters were used herein. OPT model weights were obtained from the HuggingFace Model Hub. I obtained completions from these models locally using the HuggingFace Transformers \cite{wolfHuggingFaceTransformersStateoftheart2020}
and DeepSpeed ZeRo-Inference libraries
\cite{ZeROInferenceDemocratizingMassive2022}, using a machine with a Threadripper 3960x CPU and two RTX3090 24GB GPUs. For all models, completions were produced with temperature=0 for reproducibility. The max\_tokens parameter was used to stop generation after 64 tokens (roughly 50 words). All other settings were left as default \footnote{Default values for unused parameters of the OpenAI Completions API were 
\tiny\texttt{suffix: null; top\_p: 1; n: 1; stream: false; logprobs: null; echo: false; stop: null; presence\_penalty: 0; frequency\_penalty: 0; best\_of: 1; logit\_bias: null; user: null}}.

\hypertarget{sec:text_evaluation}{%
\paragraph{Measuring Moral Content}\label{sec:text_evaluation}}

\hypertarget{moral-foundations-dictionaries}{%
\paragraph{Moral Foundations
Dictionaries}\label{moral-foundations-dictionaries}}

I estimated the moral foundational content of each completion using three dictionaries: the Moral Foundations Dictionary version 1.0 (MFDv1)
\cite{grahamLiberalsConservativesRely2009},
Moral Foundations Dictionary version 2.0 (MFDv2)
\cite{frimerMoralFoundationsDictionary2019},
the extended Moral Foundations Dictionary (eMFD)
\cite{hoppExtendedMoralFoundations2021}.

MFDv1 consists of a lexicon containing 324 word stems, with each word stem associated to one or more categories. MFDv2 consists of a lexicon of 2014 words, with each word associated to a single category. In MFDv1, the categories consist of a ``Vice'' and ``Virtue'' category for each of
the five foundations, plus a ``MoralityGeneral'' category, for 11 categories in total. MFDv2 includes all categories from MFDv1 except ``MoralityGeneral'', for a total of 10 categories. The eMFD \cite{hoppExtendedMoralFoundations2021} contains 3270 words and differs slightly from MFDv1 and MFDv2. Words in the eMFD are associated with all foundations by scores in \([0,1]\). Scores
were derived from annotation of news articles, and indicate how frequently each word was associated to each foundation, divided by the total word appearances. Word overlap between the
dictionaries is shown in Appendix Figure
\ref{fig:dictionary_word_overlap}.

\textbf{Removing Valence Information} All three dictionaries indicate whether a word is associated with the positive or negative aspect of a foundation. In MFDv1 and MFDv2 this is indicated by word association to the ``Vice'' or ``Virtue''
category for each foundation. In the eMFD, each word has sentiment scores for each foundation. In this work I was interested in the foundational contents of the completions, independent of valence. Accordingly, ``Vice'' and ``Virtue'' categories were merged into a single category for each foundation, in both MFDv1 and MFDv2. The ``MoralityGeneral'' score from MFDv1 was unused as it does not indicate association with any particular
foundation. Sentiment scores from eMFD were also unused.

\textbf{Applying the Dictionaries} Applying dictionary \(d\) to a piece
of text produces five scores
\(\{w_{df} \mkern8mu | \mkern8mu f \in F\}\). For MFDv1 and MFDv2, these are integer values representing the number of foundation-associated
words in the text. The eMFD produces continuous values in \([0,\infty]\) - the foundation-wise sums of scores for all eMFD words in the text.

I am interested in the probability \(P\) that a human or language model (apparently) expresses foundation \(f\), which I write as \(P_h(e_f)\) and \(P_{LM}(e_f)\), respectively. I use \(P^d(e_{f} | s, r, i)\) to denote this probability conditioned on a scenario \(s\), stance \(r\), and political identity \(i\), using a dictionary \(d\) for measurement.

I use \(F\) to refer to the set of moral foundations, and \(f\) for a single foundation. I use \(D\) to refer to the set of dictionaries. In each dictionary, \(W_d\) refers to all words in the dictionary. For MFDv1 and MFDv2, \(W_{df}\) refers to all the words in \(d\) belonging to foundation \(f\). I approximate \(P^d(e_{f} | s, r, i)\) as the foundation-specific score \(w_{df}\) obtained by applying the dictionary \(d\) to the model's response to a prompt, normalized by
the total score across all foundations, as shown in Equation
\ref{eq:probability_e} below.
\begin{equation}P^d(e_{f} | s, r, i) \approx \frac{w_{fd}}{\sum_{f' \in F}{w_{f'd}}}\label{eq:probability_e}\end{equation}

\hypertarget{sec:calculating_effect_sizes}{%
\paragraph{Calculating Effect
Sizes}\label{sec:calculating_effect_sizes}}

Effect sizes capture how varying political identity alters the likelihood that the model will express foundation \(f\), given the same stance and scenario. Effect sizes were calculated as the absolute difference in foundation expression probabilities for pairs of completions that differ only in political identity (Equation
\ref{eq:effect_size} below). Equation \ref{eq:average_effect_size} calculates the average effect size for foundation \(f\) over scenarios \(S\) and stances \(R\), measured by dictionary $d$. Equation \ref{eq:average_average_effect_size} gives one average effect size by the results across dictionaries.

\vspace*{-0.4\baselineskip}

\begin{equation}
\scriptstyle \Delta P^d_{i_1,i_2}(e_{f} | s, r)= P^d(e_{f} | s, i_1,  r) -  P^d(e_{f}  |  s, i_2,  r)\label{eq:effect_size}\end{equation}


\vspace*{-\baselineskip}

\begin{equation} 
\scriptstyle \Delta P^d_{i_1,i_2}(e_{f}) = \mathop{\mathbb{E}}_{s, r \in S \times R}  \Delta P^d_{i_1,i_2}(e_{f} | s, r)\label{eq:average_effect_size}
\end{equation}

\vspace*{-\baselineskip}

\begin{equation} 
\scriptstyle \Delta P_{i_1,i_2}(e_{f}) =  \mathop{\mathbb{E}}_{d \in D}  \Delta P^d_{i_1,i_2}(e_{f})\label{eq:average_average_effect_size}\end{equation}

\hypertarget{sec:methods_results_2}{%
\subsection{LLM vs. Human Moral Foundation
Use}\label{sec:methods_results_2}}

\hypertarget{experiment-details}{%
\paragraph{Experiment Details}\label{experiment-details}}

This experiment considers whether LLMs use foundation words that are situationally appropriate\footnote{e.g. using the Care/Harm foundation when prompted with a violent scenario}. LLMs would satisfy a weak criterion
for this capability if they were more likely to express foundation \(f\) in response to scenarios where foundation \(f\) is salient, compared to their average use of \(f\) across a corpus of scenarios containing all foundations in equal proportion. I formalize this with Criterion A
below.

\textbf{Criterion A} Average use of foundation \(f\) is greater across scenarios \(S_f\) that demonstrate only foundation \(f\), in comparison
to average use of foundation \(f\) across a foundationally-balanced corpus of scenarios \(S\) (Equation \ref{eq:hypothesis_2_weak}).
\begin{equation} 
\begin{aligned}
\scriptstyle \mathop{\mathbb{E}}_{s_f, r \in S_f \times R} P_{LM}(e_{f}|s_f,r) > \mathop{\mathbb{E}}_{s, r \in S \times R} P_{LM}(e_{f} |s, r) \nonumber
\end{aligned}
\label{eq:hypothesis_2_weak}\end{equation}

A stronger criterion would require LLMs to not to deviate from human foundation use beyond some level of variation that is expected among humans. I formalize this with Criterion 2b below.

\textbf{Criterion B} The average difference between language model and consensus human foundation use is less than the average difference between individual human and consensus human foundation use.

\begin{equation}
\scriptstyle \textsc{Diff}_{LM,C_H} \leq \textsc{Diff}_{H,C_H}
\end{equation}

\vspace*{-\baselineskip}
\begin{equation}
\scriptstyle \textsc{Diff}_{LM,C_H} = \mathop{\mathbb{E}}_{s \in S} & \left[  |P_{LM}(e_{f} | s, r_{H}(s)) - C_{H}(s)|\right] 
\end{equation}

\vspace*{-\baselineskip}
\begin{equation}
\scriptstyle \textsc{Diff}_{H,C_H} = \mathop{\mathbb{E}}_{s \in S} \left[\mathop{\mathbb{E}}_{H}\left[|P_{h}(e_{f}|s) - C_{H}(s)|\right]\right]
\end{equation}

\vspace*{-\baselineskip}
\begin{equation}
\scriptstyle C_{H}(s)= \mathop{\mathbb{E}}_h[P_{h}(e_{f}|s)]
\end{equation}

Stance \(r_{Hs}\) is the normative moral acceptability of scenario \(s\) - the human-written rationalizations are ``conditioned'' on human
normative stance for each scenario, so I only compare these with model outputs that are also conditioned on human normative stance.

Criterion A requires a corpus with ground-truth knowledge that only a particular foundation \(f\) is salient for each scenario. To obtain such clear-cut scenarios, I select the least ambiguous actions from the Social Chemistry dataset, according to the filtering methods described in Appendix Section \ref{sec:social_chemistry_actions}. Estimating human consensus foundation use (Criterion B) requires a corpus of scenarios that are each annotated in open-ended fashion by
multiple humans. I obtain such a corpus from the Social Chemistry dataset using the methods described in Appendix Section \ref{sec:social_chemistry_situations}.

\hypertarget{results}{%
\paragraph{Results}\label{results}}

\leavevmode\hypertarget{fig:elephants}{}%
\begin{figure}
\hypertarget{fig:moral_foundation_vs_ground_truth}{%
\centering
\includegraphics[width=0.45\textwidth]{figures/rq2_agg2.png}
\caption{Left: Foundation expression probabilities for foundation-specific
examples vs.~average foundation use across all examples. Text-davinci-002; Social Chemistry Actions
scenarios. Right: LM and individual human differences from human consensus
foundation use, in response to scenarios from the Social Chemistry
Situations dataset; text-davinci-002.}
\label{fig:moral_foundation_vs_ground_truth}
}
\end{figure}


\begin{figure*}
\hypertarget{fig:main_results}{%
\centering
\includegraphics[width=\textwidth]{figures/results_by_dictionary_normalized_alt_davinci_001.png}
\caption{Effect sizes for liberal vs.~conservative political identity
for OPT-30B, text-davinci-001, text-davinci-002, and text-davinci-003.
Dot markers represent average effect size. Error bars represent 95\% CI.
Shaded regions represent directions of expected effect size based on the
Moral Foundations Hypothesis.}\label{fig:main_results}
}
\end{figure*}


Figure \ref{fig:moral_foundation_vs_ground_truth} (left) shows average values of \(P(e_{f}|s)\) for each foundation. For all five foundations, the model increases its apparent use of foundation-associated words appropriate to the ground truth foundation label, satisfying Criterion A. Figure \ref{fig:moral_foundation_vs_ground_truth} (right) shows LM differences from human consensus \(|P_{LM}(e_{f}| s, r_{Hs}) - C_{H}(s)|\) obtained from the text-davinci-002 model, and human differences from human consensus
\(\mathbb{E}_{H}\left[|P_{h}(e_{f}|s) - C_{H}(s)|\right]\), on the Social Chemistry Situations dataset. In general the LM-human differences are greater than the human-human differences.

\hypertarget{sec:methods_results_3}{%
\subsection{Are LLMs Moral Mimics?}\label{sec:methods_results_3}}

\hypertarget{experiment-details-1}{%
\paragraph{Experiment Details}\label{experiment-details-1}}

I consider whether conditioning LLMs with political identity influences
their use of moral foundations in a way that reflects human moral
biases. To investigate this question I used a corpus of 2,000 scenarios
obtained from the Moral Stories dataset and 1,000 scenarios obtained
from the ETHICS dataset, described in Appendix Section
\ref{sec:additional_dataset_information}.

Prompts were constructed with template style 2 from table
\ref{tbl:prompt_styles_situations}. For each scenario, four prompts were
constructed based on combinations of ``liberal'' and ``conservative''
political identity and moral and immoral stance, for a total of 12,000
prompts. Completions were obtained from the most capable model in each
family that our computational resources afforded: text-davinci-001
(GPT-3), text-davinci-002 and text-davinci-003 (GPT-3.5) and OPT-30B.
One generation was obtained from each model for each prompt. I
calculated average effect size \(\Delta P_{i_1,i_2}(e_{f})\) with
\(i_1\) = ``liberal'' and \(i_2\) = ``conservative'' for all five
foundations. Effect sizes were computed separately for each dictionary,
for a total of 18,000 effect sizes computed per model.

\hypertarget{results-1}{%
\paragraph{Results}\label{results-1}}

Figure \ref{fig:main_results} shows effect sizes for liberal
vs.~conservative political identity, for the most capable models tested
from the OPT, GPT, and GPT-3.5 model families, measured using the three
moral foundations dictionaries. The shaded regions in each plot
represent the effects that would be expected based on the Moral
Foundations Hypothesis - namely that prompting with liberal political
identity would result in more use of the individualizing foundations
(positive \(\Delta P_{i_1,i_2}\)) and prompting with conservative
political identity would result in more use of the binding foundations
(negative \(\Delta P_{i_1,i_2}\)).

The majority of effect sizes coincide with the Moral Foundations
Hypothesis. Of 60 combinations of 5 foundations, 4 models, and 3
dictionaries, only 11 effect sizes are in the opposite direction from
expected, and all of these effect sizes have magnitude of less than 1
point absolute difference.

\hypertarget{sec:methods_results_6}{%
\subsection{Is Moral Mimicry Affected By Model
Size?}\label{sec:methods_results_6}}

\hypertarget{experiment-details-2}{%
\paragraph{Experiment Details}\label{experiment-details-2}}

In this section, I consider how moral mimicry relates to model size. I
used text-ada-001, text-babbage-001, text-curie-001, and
text-davinci-001 models from the GPT-3 family, text-davinci-002 and
text-davinci-003 from the GPT-3.5 family
\cite{openaiModelIndexResearchers}, and
OPT-350m, OPT-1.3B, OPT-6.7B, OPT-13B, and OPT-30B
\cite{zhangOPTOpenPretrained2022}. The GPT-3
models have estimated parameter counts of 350M, 1.3B, 6.7B and 175B,
respectively
\cite{openaiModelIndexResearchers,gaoSizesOpenAIAPI2021}.
Text-davinci-002 and text-davinci-003 also have 175B parameters
\cite{openaiModelIndexResearchers}.
Parameters in billions for the OPT models are indicated in the model
names.

To analyze to what extent each model demonstrates the moral mimicry
phenomenon, I define a scoring function \textsc{MFH-Score} that scores
a model \(m\) as follows:

\vspace*{-0.5\baselineskip}

\begin{equation}
\scriptstyle \textsc{MFH-Score}(m) = \sum_{f \in F} \operatorname{sign}_{MFH}(f) \Delta P_m\left(e_f\right)
\end{equation}

\vspace*{-\baselineskip}

\begin{equation}
\scriptstyle \operatorname{sign}_{MFH} = \begin{cases}
-1, & \text{if} f \in \{\text{A/S, S/D, L/B}\} \\ \\
+1, & \text{if} f \in \{\text{C/H, F/C}\} 
\end{cases} 
\end{equation}


\begin{equation*}
\scriptstyle \text{A/S: Authority/Subversion; S/D: Sanctity/Degradation;  }\\ 
\end{equation*}

\vspace*{-1.8\baselineskip}

\begin{equation*}
\scriptstyle \text{
L/B: Loyalty/Betrayal; C/H: Care/Harm; F/C; Fairness/Cheating}
\end{equation*}


The \textsc{MFH-Score} calculates the average effect size for each
model in the direction predicted by the Moral Foundations Hypothesis.

\hypertarget{results-2}{%
\paragraph{Results}\label{results-2}} Figure \ref{fig:effect_size_vs_scale_agg} above shows effect sizes \(\Delta(P_{e_f})\) for each foundation and \textsc{MFH-Score}s vs.~model size (number of parameters). Effect sizes are averaged over the three moral foundations dictionaries.

For the OPT model family, we can see that model parameter count and \textsc{MFH-Score} show some relationship (\textit{r}=0.69, although statistical power is limited due to the limited number of models). In particular, the Sanctity/Degradation foundation maintains a non-zero effect size in the expected direction for all models 6.7B parameters or larger. Surprisingly, OPT-13B shows decreased effect sizes for Fairness/Cheating and Care/Harm in comparison to the smaller OPT-6.7B. The relationship between model size and effect size is weaker for GPT-3 (\textit{r}=0.23). Care/Harm, Fairness/Cheating, Sanctity/Degradation, and
Authority/Subversion have effect size in the expected direction for Babbage, Curie, and DaVinci models, though the effect sizes are smaller
than for the OPT family. Models from the GPT-3.5 family show the largest effect sizes overall. Unfortunately, no smaller model sizes are available for this family. If we include the GPT-3 and GPT-3.5 models together (indicated by
$\dagger$ in Figure \ref{fig:effect_size_vs_scale_agg}), the
correlation between \textsc{MFH-Score} and model parameters increases
to \textit{r}=0.84. Interestingly, the OPT and GPT-3 families show Sanctity/Degradation as the most pronounced effect size for conservative prompting, and Fairness/Cheating as the most pronounced effect size for liberal prompting. GPT-3.5 instead shows the largest effect sizes for Authority/Subversion and Care/Harm, respectively.




\begin{figure*}
\hypertarget{fig:effect_size_vs_scale_agg}{%
\centering
\includegraphics[width=\textwidth]{figures/effect_sizes_vs_scale_moral_stories_agg.png}
\caption{Top: Effect size vs.~model parameters, based on completions
obtained from Moral Stories dataset. Dark lines show mean effect size.
Error bars show 95\% CI. Effect sizes are averaged over the three moral
foundations dictionaries.; 002: text-davinci-002; 003:
text-davinci-003.; Bottom: \textsc{MFH-Score} vs.~model parameters; r,p: value
and p-value for Pearson's Correlation between \textsc{MFH-Score} and model
parameters.; †results of correlation analysis with GPT-3 and GPT-3.5
models analyzed together}\label{fig:effect_size_vs_scale_agg}
}
\end{figure*}


\hypertarget{sec:discussion}{%
\section{Discussion}\label{sec:discussion}}

Section \ref{sec:methods_results_2} posed two criteria to judge whether
LLMs use moral foundations appropriately. For the weaker Criterion A,
results show that LLMs do increase use of foundation words relevant to
the foundation that is salient in a given scenario, at least for
scenarios with clear human consensus on foundation salience. However,
for Criterion B, results show that LLMs differ more from human consensus
foundation use than humans do in terms of foundation use.

Section \ref{sec:methods_results_3} compared LM foundation use with
findings from moral psychology that identify differences in the moral
foundations used by liberal and conservative political groups.
Specifically, according to the Moral Foundations Hypothesis, liberals
rely mostly on the Care/Harm and Fairness/Cheating foundations, while
conservatives use all 5 foundations more evenly, using
Authority/Subversion, Loyalty/Betrayal, and Fairness/Cheating more than
liberals. This finding was first presented in
\cite{grahamLiberalsConservativesRely2009},
and has since been supported with confirmatory factor analysis in
\cite{dogruyolFivefactorModelMoral2019}, and
partially replicated (though with smaller effect sizes) in
\cite{frimerLiberalsConservativesUse2020}.

Results indicate that models from the GPT-3, GPT-3.5 and OPT model families are more likely to use the binding foundations when prompted with conservative political identity, and are more likely to use the individualizing foundations when prompted with liberal political identity. Emphasis on individual foundations in each category differs by model family. OPT-30B shows larger effect sizes for Fairness/Cheating than Care/Harm and larger effect sizes for Sanctity/Degradation vs.~Authority/Subversion, while GPT-3.5 demonstrates the opposite. I suspect that this may be due to differences in training data and/or training practices between the model families. This opens an interesting question of how to influence the moral mimicry capabilities that emerge during training, via dataset curation or other methods.  

The results from Section \ref{sec:methods_results_6} show some relationship between moral mimicry and model size. Effect sizes tend to increase with parameter count in the OPT family, and less so in the GPT-3 family. Both 175B-parameter GPT-3.5 models show relatively strong moral mimicry capabilities, moreso than the 175B GPT-3 model text-davinci-001. This suggests that parameter count is not the only factor leading to moral mimicry. The GPT-3.5 models were trained with additional supervised fine-tuning not applied to the GPT-3 family, and used text and code pre-training rather than text alone \cite{openaiModelIndexResearchers}. 

\hypertarget{limitations}{%
\section{Limitations}\label{limitations}}

This work used the moral foundations dictionaries to measure the moral content of text produced by GPT-3. While studies have demonstrated correspondence between results from the dictionaries and human labels of moral foundational content \cite{mutluBotsHaveMoral2020, grahamLiberalsConservativesRely2009}, dictionary-based analysis is limited in its ability to detect nuanced moral expressions. Dictionary-based analysis could be complemented with machine-learning approaches \cite{gartenMoralityLinesDetecting2016, johnsonClassificationMoralFoundations2018, pavanMoralityClassificationNatural2020,royFewShotIdentificationMorality2022} as well as human evaluation.  This study attempted to control for variations in the prompt phrasing by averaging results over several prompt styles (Tables \ref{tbl:prompt_styles_situations} and \ref{tbl:prompt_styles_actions}). These prompt variations were chosen by the author. A more principled selection procedure could result in a more diverse set of prompts. The human studies that this study refers to \cite{grahamLiberalsConservativesRely2009,frimerLiberalsConservativesUse2020} were performed on populations from the United States. The precise political connotations of the terms ``liberal'' and ``conservative'' differ across demographics. Future work may explore how language model output varies when additional demographic information is provided, or when multilingual models are used.  Documentation for the datasets used herein indicates that the crowd workers leaned politically left, and morally towards the Care/Harm and Fairness/Cheating foundations \cite{forbesSocialChemistry1012021,hendrycksAligningAIShared2021,fraserDoesMoralCode2022}. However, bias in the marginal foundation distribution does not hinder the present analysis, since the present experiments experiments focus primarily on the difference in foundation use resulting from varying political identity. The analysis in Section \ref{sec:methods_results_2} relies more heavily on the marginal foundation distribution; a foundationally-balanced dataset was constructed for this experiment.  This study used GPT-3 \cite{brownLanguageModelsAre2020}, GPT-3.5 \cite{openaiModelIndexResearchers}, and OPT \cite{zhangOPTOpenPretrained2022}. Other pre-trained language model families of similar scale and architecture include BLOOM\footnote{https://bigscience.huggingface.co/blog/bloom}, which I was unable to test due to compute budget, and LLaMA \cite{touvronLLaMAOpenEfficient2023}, which was released after the experiments for this work concluded. While the OPT model weights are available for download, GPT-3 and GPT-3.5 model weights are not; this may present barriers to future work that attempts to connect the moral mimicry phenomenon to properties of the model. On the other hand, the hardware required to run openly-available models may be a barrier to experimentation that is not a concern for models hosted via an API.  

Criticisms of Moral Foundations Theory include disagreements about whether a pluralist theory of morality is parsimonious \cite{suhlerCanInnateModular2011,dobolyiCritiquesMoralFoundations2016}; Ch. 6 of \cite{haidtRighteousMindWhy2013}, disagreements about the number and character of the foundations \cite{yalcindagInvestigationMoralFoundations2019,harperReanalysingFactorStructure2021}, disagreements about stability of the foundations across cultures \cite{davisMoralFoundationsHypothesis2016}, and criticisms suggesting bias in the Moral Foundations Questionnaire \cite{dobolyiCritiquesMoralFoundations2016}. Moral foundations theory was used in this study because it provides established methods to measure moral content in text, and because MFT-based analyses have identified relationships between political affiliation and moral biases, offering a way to compare LLM and human behavior. The methods presented here may be applicable to other theories of morality; this is left for future work.

Work that aims to elicit normative moral or ethical judgement from non-human systems has received criticism. Authors have argued that non-human systems lack the autonomy and communicative intent to be moral agents \cite{talatMachineLearningEthical2022,benderClimbingNLUMeaning2020}. Criticisms have also been raised about the quality and appropriateness of data used to train such systems. Notably, crowdsourced or repurposed data often reflects \textit{a priori} opinions of individuals who may not be informed about the topics they are asked to judge, and who may not have had the opportunity for discourse or reflection before responding \cite{talatMachineLearningEthical2022,etienneDarkSideMoral2021}. Some have argued that systems that aggregate moral judgements from descriptive datasets cannot help but be seen as normative, since their reproduction of the popular or average view tends to be implicitly identified with a sense of correctness \cite{talatMachineLearningEthical2022}. Finally, several authors argue that the use of non-human systems that produce apparent or intended normative judgements sets a dangerous precedent by short-circuiting the discursive process by which moral and ethical progress is made, and by obscuring accountability should such a system cause harm \cite{talatMachineLearningEthical2022,etienneDarkSideMoral2021}. 

The present study investigates the apparent moral rationalizations produced by prompted LLMs. This study does not intend to produce a system for normative judgement, and I would discourage a normative use or interpretation of the methods and results presented here. The recent sea change in natural language processing towards general-purpose LLMs prompted into specific behaviors enables end users to produce a range of outputs of convincing quality, including apparent normative moral or ethical judgements. Anticipating how these systems will impact end users and society requires studying model behaviors under a variety of prompting inputs. The present study was conducted with this goal in mind, under the belief that the benefit of understanding the moral mimicry phenomenon outweighs the risk of normative interpretation.

\hypertarget{related-work}{% 
\section{Related Work}\label{related-work}}  

Several machine ethics projects have assessed the extent to which LLM-based systems can mimic human normative ethical judgement, for example \cite{hendrycksAligningAIShared2021} and \cite{jiangCanMachinesLearn2021}. Other projects evaluate whether LLMs can produce the relevant moral norms for a given scenario \cite{forbesSocialChemistry1012021,emelinMoralStoriesSituated2021}, or whether they can determine which scenarios justify moral exceptions \cite{jinWhenMakeExceptions2022}. Yet other works focus on aligning models to normative ethics \cite{ziemsMoralIntegrityCorpus2022}, and investigating to what extent societal biases are reproduced in language models (see Section 5.1 of \citealt{bommasaniOpportunitiesRisksFoundation2022}). As an example, \citeauthor*{fraserDoesMoralCode2022} (2022) analyze responses of the Delphi model \cite{jiangCanMachinesLearn2021} to the Moral Foundations Questionnaire \cite{grahamMappingMoralDomain2011}, finding that its responses reflect the moral foundational biases of the groups that produced the model and its training data.  

The aforementioned research directions typically investigate language models not prompted with any particular identity. This framing implies the pre-trained model itself as the locus where a cohesive set of biases might exist. Recent work suggests an alternative view that a single model may be capable of simulating a multitude of ``identities'', and that these apparent identities may be selected from by conditioning the model via prompting \cite{argyleOutOneMany2023,aherUsingLargeLanguage2023}. Drawing on the latter view, the present study prompts LLMs to simulate behavior corresponding to opposed political identities, and evaluates the fidelity of these simulacra with respect to moral foundational bias. Relations between the present work and other works taking this ``simulation'' view are summarized below.  

Arora et. al. probe for cultural values using Hofstede's six-dimenension theory \cite{hofstedeCultureRecentConsequences2001} and the World Values Survey \cite{worldvaluessurveyWVSDatabase2022}, and use prompt language rather than prompt tokens to condition the model with a cultural ``identity''. \citealt{alshomaryBeliefbasedGenerationArgumentative2021} and \citealt{qianMoralityLinesDetecting2021} fine-tune GPT-2 models (1.5B parameters) on domain-specific corpora, and condition text generation with stances on social issues. The present work, in contrast, conditions on political identity rather than stance, evaluates larger models without domain-specific fine-tuning, and investigates LLM capabilities to mimic moral preferences. 
Concurrent work probes language models for behaviors including \textit{sycophancy}, the tendency to mirror users' political views in a dialog setting \cite{perezDiscoveringLanguageModel2022}. Perez \textit{et. al.} find that this tendency increases with scale above \textasciitilde10B parameters. While sycophancy describes how model-generated text appears to express political views, conditioned on dialog user political views, moral mimicry describes how model-generated text appears to express moral foundational salience, conditioned on political identity labels. Argyle et. al.~propose the concept of ``algorithmic fidelity'' - an LLM's ability to ``accurately emulate the response distribution \ldots{} of human subgroups'' under proper conditioning \cite{argyleOutOneMany2023}. Moral mimicry can be seen as an instance of algorithmic fidelity where moral foundation use is the response variable of interest. Argyle et. al.~study other response variables: partisan descriptors, voting patterns, and correlational structure in survey responses.



\hypertarget{conclusion}{%
\section{Conclusion}\label{conclusion}}

This study evaluates whether LLMs can reproduce the moral foundational biases associated with social groups, a capability herein coined \emph{moral mimicry}. I measure the apparent use of five moral foundations in the text generated by pre-trained language models conditioned with a political identity. I show that LLMs reproduce the moral foundational biases associated with liberal and conservative political identities, modify their moral foundation use situationally, although not indistinguishably from humans, and that moral mimicry may relate to model size. 

\hypertarget{acknowledgements}{%
\section*{Acknowledgements}\label{acknowledgements}}

I would like to thank the anonymous reviewers who provided valuable comments on this paper. I would also like to thank Professors Dipak Ghosal, Jiawei Zhang, and Patrice Koehl, who provided valuable feedback on this work, and colleagues, friends, and family for insightful discussions.

% Entries for the entire Anthology, followed by custom entries
\bibliography{library}
\bibliographystyle{acl_natbib}


\newpage
\appendix

\hypertarget{appendix-a-additional-details-related-to-experimental-methods}{%
\section{Appendix A: Additional Details Related to Experimental
Methods}\label{appendix-a-additional-details-related-to-experimental-methods}}

\hypertarget{sec:additional_details_models}{%
\subsection{Additional Details Related to LLMs Used in the
Study}\label{sec:additional_details_models}}

\begin{table}[!ht]
    \centering
    \resizebox{0.5\textwidth}{!}{%
    \begin{tabular}{|l|l|l|l|}
    \hline
        Model Family & Model Variant & Number of Parameters & Instruction Fine-tuning \\ \hline
        GPT-3 & text-ada-001 & 350M & None \\ \hline
        GPT-3 & text-babbage-001 & 1.3B & FeedME \\ \hline
        GPT-3 & text-curie-001 & 6.7B & FeedME \\ \hline
        GPT-3 & text-davinci-001 & 175B & FeedME \\ \hline
        GPT-3.5 & text-davinci-002 & 175B & ? \\ \hline
        GPT-3.5 & text-davinci-003 & 175B & PPO \\ \hline
        OPT & opt-350m & 350M & None \\ \hline
        OPT & opt-1.3b & 1.3B & None \\ \hline
        OPT & opt-6.7b & 6.7B & None \\ \hline
        OPT & opt-13b & 13B & None \\ \hline
        OPT & opt-30b & 30B & None \\ \hline
    \end{tabular}}
    
\caption{Models evaluated in this study. Information for GPT-3 and
GPT-3.5 from
\cite{openaiModelIndexResearchers}.
Information for OPT from
\cite{zhangOPTOpenPretrained2022}.
Information for OPT-IML from
\cite{iyerOPTIMLScalingLanguage2023}. FeedME:
``Supervised fine-tuning on human-written demonstrations and on model
samples rated 7/7 by human labelers on an overall quality score''
\cite{openaiModelIndexResearchers}; PPO:
``Reinforcement learning with reward models trained from comparisons by
humans'' \cite{openaiModelIndexResearchers};
?: use of instruction fine-tuning is uncertain based on documentation.
\label{tbl:model_details}}

\end{table}

\hypertarget{sec:additional_dataset_information}{%
\subsection{Additional Details Related to Datasets Used in the
Study}\label{sec:additional_dataset_information}}

\hypertarget{sec:moral_stories}{%
\subsubsection{Preprocessing Details for Moral Stories
Dataset}\label{sec:moral_stories}}

Each example in Moral Stories consists of a \emph{moral norm} (a
normative expectation about moral behavior), a \emph{situation} which
describes the state of some characters, an \emph{intent} which describes
what a particular character wants, and two \emph{paths}, a \emph{moral
path} and \emph{immoral path}. Each path consists of a \emph{moral} or
\emph{immoral action} (an action following or violating the norm) and a
\emph{moral} or \emph{immoral consequence} (a likely outcome of the
action). For the present experiments, I construct scenarios as the
string concatenation of an example's situation, intent, and either moral
action or immoral action. We do not use the consequences or norms, as
they often include a reason why the action was moral/immoral, and thus
could bias the moral foundational contents of the completions.

We used 2,000 scenarios produced from the Moral Stories dataset,
consisting of 1,000 randomly-sampled moral scenarios and 1,000
randomly-sampled immoral scenarios.

\hypertarget{sec:ETHICS}{%
\subsubsection{Preprocessing Details for ETHICS
Dataset}\label{sec:ETHICS}}

The ETHICS dataset contains five subsets of data, each corresponding to
a particular ethical framework (deontology, justice, utilitarianism,
commonsense, and virtue), each further divided into a ``train'' and
``test'' portion. For the present experiments, I use the ``train''
split of the ``commonsense'' portion of the dataset, which contains
13,910 examples of scenarios paired with ground-truth binary labels of
ethical acceptability. Of these, 6,661 are ``short'' examples, which are
1-2 sentences in length. These short examples were sourced from Amazon
Mechanical Turk workers and consist of 3,872 moral examples, and 2,789
immoral examples. From these, I randomly select 1,000 examples split
evenly according to normative acceptability, resulting in 500 moral
scenarios and 500 immoral scenarios. The train split of the commonsense
portion of the ETHICS dataset also contains 7,249 ``long'' examples, 1-6
paragraphs in length, which were obtained from Reddit. These were unused
in the present experiment, primarily due to the increased costs of using
longer scenarios.

\hypertarget{sec:social_chemistry_actions}{%
\subsubsection{Preprocessing Details for Social Chemistry Actions
Dataset}\label{sec:social_chemistry_actions}}

The Social Chemistry 101
\cite{forbesSocialChemistry1012021} dataset
contains 355,922 structured annotations of 103,692 situations, drawn
from four sources (Dear Abby, Reddit AITA, Reddit Confessions, and
sentences from the ROCStories corpus; see
\cite{forbesSocialChemistry1012021} for
references). Situations are brief descriptions of occurrences in
everyday life where social or moral norms may dictate behavior, for
example ``pulling out of a group project at the last minute''.
Situations are annotated with Rules-of-Thumb (RoTs), which are
judgements of actions that occur in the situation, such as ``It's bad to
not follow through on your commitments''. Some situations may contain
more than one action, but I consider situations that are unanimously
annotated as having only one action for the present experiment, as this
simplifies interpretation of the moral foundation annotations. RoTs in
the dataset are annotated with ``RoT breakdowns''. RoT breakdowns parse
each RoT into its constituent action (e.g.~``not following through on
commitments'') and judgement (``it's bad''). Judgements are standardized
to five levels of approval/disapproval:
very bad, bad, expected/OK, good, very good. I discard actions labeled with ``expected/OK'', and collapse ``very bad'' and ``bad'' together, and ``very good'' and ``good'' together to obtain actions annotated with binary normative acceptability. Actions are also annotated with moral foundation labels (the example in the previous sentence was annotated with the Fairness/Cheating and Loyalty/Betrayal foundations). Additionally, each RoT belongs to one of the following categories - morality-ethics, social-norms, advice, description. I use
RoTs belonging to the ``morality-ethics'' category, since this is the category indicating that the RoT contains moral reasoning rather than advice or etiquette recommendations. After filtering RoTs and situations by category, and selecting examples with unanimous ratings for moral foundation and normative acceptability, I obtain a dataset of 1300
actions - 130 normatively moral actions and 130 normatively immoral actions for each of the five moral foundations. These scenarios are used in the experiment related to Criterion A in Section
\ref{sec:methods_results_2}.

\hypertarget{sec:social_chemistry_situations}{%
\subsubsection{Preprocessing Details for Social Chemistry Situations
Dataset}\label{sec:social_chemistry_situations}}

Criterion B requires comparing \(P_H(e_{f} | s)\) and
\(P_{LM}(e_{f} | s)\), for human- and LLM-written open-ended text
responses containing moral reasoning about some scenarios. I use
situations from the Social Chemistry 101 dataset
\cite{forbesSocialChemistry1012021}, and use
the human-written RoTs to estimate \(P_H(e_{f}|s)\) using the moral
foundations dictionaries. To estimate consensus human judgement
\(C_H(s)\), I use situations that are multiply annotated. Specifically,
I filter the Social Chemistry 101 dataset to situations with 4 or more
RoTs, and 4 or more RoT breakdowns per RoT. This results in a corpus of
170 scenarios. Unlike the Social Chemistry Actions dataset, this Social
Chemistry Situations dataset is not foundationally-balanced - I
encountered a tradeoff between the minimum number of annotations per
situation, and the final corpus size - balancing the dataset in terms of
foundations would have reduced the dataset size further. The set of
scenarios is used for the experiment related to Criterion B in Section
\ref{sec:methods_results_2}.

\hypertarget{sec:appendix_mfd}{%
\subsection{Additional Details Related to Moral Foundations
Dictionaries}\label{sec:appendix_mfd}}

\begin{figure}
\hypertarget{fig:dictionary_word_overlap}{%
\centering
\includegraphics{figures/mfd_venn_diagram.png}
\caption{Venn diagram of word overlap between MFDv1, MFDv2 and eMFD.
Since some entries in MFDv1 are regexes, I represent MFDv1 in this
diagram by all non-compound words in WordNet matching a regex in
MFDv1.}\label{fig:dictionary_word_overlap}
}
\end{figure}

\hypertarget{sec:additional_details_prompt_construction}{%
\subsection{Additional Details Related to Prompt
Construction}\label{sec:additional_details_prompt_construction}}

Templates from Table \ref{tbl:prompt_styles_situations} were used for the Moral Stories, ETHICS, and Social Chemistry Situations datasets, where the scenarios are longer descriptions of events, with length one sentence or
longer. Templates from Table \ref{tbl:prompt_styles_actions} were used
for the Social Chemistry Actions dataset, where scenarios are brief action descriptions (sentence fragments). This was done to ensure grammaticality.


\begin{table}[!ht]
    \centering
    \resizebox{0.5\textwidth}{!}{%
    \begin{tabular}{|l|l|l|}
    \hline
        Template Style & Stance & Template \\ \hline
        1 & moral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], this is a moral thing to do because \\ \hline
        1 & immoral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], this is an immoral thing to because \\ \hline
        2 & moral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here are the moral arguments for why this is a moral thing to do: \\ \hline
        2 & immoral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here are the moral arguments for why this is an immoral thing to do: \\ \hline
        3 & moral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why this is acceptable behavior: \\ \hline
        3 & immoral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why this is unacceptable behavior: \\ \hline
        4 & moral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why this is a good thing to do: \\ \hline
        4 & immoral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why this is a bad thing to do: \\ \hline
        5 & moral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why this is the right thing to do: \\ \hline
        5 & immoral & [SCENARIO] As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why this is the wrong thing to do: \\ \hline
    \end{tabular}}
    \caption{Prompt template styles for situations
\label{tbl:prompt_styles_situations}}

\end{table}

\begin{table}[!ht]
    \centering
    \resizebox{0.5\textwidth}{!}{%
    \begin{tabular}{|l|l|l|}
    \hline
        Template Style & Stance & Template \\ \hline
        1 & moral & As a/an [POLITICAL\_IDENTITY\_PHRASE], [ACTION] is a moral thing to do because \\ \hline
        1 & immoral & As a/an [POLITICAL\_IDENTITY\_PHRASE], [ACTION] is an immoral thing to because \\ \hline
        2 & moral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here are the moral arguments for why [ACTION] is a moral thing to do: \\ \hline
        2 & immoral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here are the moral arguments for why [ACTION] is an immoral thing to do: \\ \hline
        3 & moral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why [ACTION] is acceptable behavior: \\ \hline
        3 & immoral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why [ACTION] is unacceptable behavior: \\ \hline
        4 & moral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why [ACTION] is a good thing to do: \\ \hline
        4 & immoral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why [ACTION] is a bad thing to do: \\ \hline
        5 & moral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why [ACTION] is the right thing to do: \\ \hline
        5 & immoral & As a/an [POLITICAL\_IDENTITY\_PHRASE], here's why [ACTION] is the wrong thing to do: \\ \hline
    \end{tabular}}
    \caption{Prompt template styles for actions
\label{tbl:prompt_styles_actions}}\tabularnewline
\toprule
\end{table}

\hypertarget{appendix-b-additional-experimental-results}{%
\section{Appendix B: Additional Experimental
Results}\label{appendix-b-additional-experimental-results}}

\hypertarget{effect-size-vs.-dataset}{%
\subsection{Effect Size vs.~Dataset}\label{effect-size-vs.-dataset}}

\begin{figure}
\hypertarget{fig:results_by_dataset}{%
\centering
\includegraphics[width=0.5\textwidth]{figures/dataset_effect_size_normalized_absolute_bar_style_2.png}
\caption{Effect sizes, liberal vs.~conservative prompt identity, by
dataset and dictionary}\label{fig:results_by_dataset}
}
\end{figure}

Figure \ref{fig:results_by_dataset} shows effect sizes for liberal
vs.~conservative prompting, based on completions obtained from 2000
scenarios produced from Moral Stories and 1000 scenarios produced from
ETHICS. Scores are separated by dictionary and dataset. See Section
\ref{sec:calculating_effect_sizes} for the methods used to calculate
effect sizes.

Effect sizes and directions are consistent across datasets for the
Care/Harm and Authority/Subversion foundations.

\hypertarget{sec:results_vs_prompt_style}{%
\subsection{Effect Size vs.~Prompt Template
Style}\label{sec:results_vs_prompt_style}}

\begin{figure}
\hypertarget{fig:results_vs_prompt_style}{%
\centering
\includegraphics[width=0.5\textwidth]{figures/prompt_sensitivity_effect_size_normalized_absolute_bar.png}
\caption{Effect sizes, liberal vs.~conservative prompt identity, by
prompt style and dictionary.}\label{fig:results_vs_prompt_style}
}
\end{figure}

Figure \ref{fig:results_vs_prompt_style} shows the results obtained from
analysis of compeletions obtained from five different prompt styles, as
described in \ref{tbl:prompt_styles_situations}.

Effects of liberal vs.~conservative political identity are uniform in
direction for the Care/Harm and Authority/Subversion foundations.
Regardless of the prompt style or dictionary used, the completions
contain more Care/Harm words when the liberal political identity is
used, and more Authority/Subversion words when the conservative
political identity is used. Effects are nearly uniform in direction for
the Fairness/Cheating foundation, with liberal political identity
resulting in increased use of this foundation for thirteen of fifteen
combinations of prompt style and dictionary. Liberal prompting resulted
in decreased use of the Fairness/Cheating foundation for prompt styles 1
and 2, when measured using MFDv2.

Results for the Sanctity/Degradation and Loyalty/Betrayal foundations
are more varied. Effect directions are uniform for the
Sanctity/Degradation foundation when measured with MFDv2 - liberal
political identity results in lower Sanctity/Degradation use by 1-2
percent score across all prompt styles. Effects on Sanctity/Degradation
are less consistent when measured using MFDv1 or eMFD - liberal
prompting resulted in decreased use of Sanctity/Degradation words for
only three out of five prompt styles. Measured by the eMFD, liberal
prompting results in decreased use of Sanctity/degradation words for
four of five prompt styles.

Effect directions are uniform for Loyalty/Betrayal when measured with
MFDv1 - prompting with liberal political identity results in greater
percent scores for Loyalty for all prompt styles. Results are varied
when measured with MFDv1 - liberal prompting results in decreased use
for only three of five prompt styles. When measured using the eMFD,
liberal prompting results in decreased or equal use of the
Loyalty/Betrayal foundation across the prompt styles, which is
consistent within the dictionary, but is opposite in effect direction in
comparison to MFDv1.


\onecolumn
\hypertarget{appendix-c-llm-output-examples}{%
\section{Appendix C: LLM Output
Examples}\label{appendix-c-llm-output-examples}}
\begin{figure}[!htb]
\hypertarget{fig:scale_examples_gpt}{%
\centering
\includegraphics[width=\textwidth]{figures/all.png}
\caption{Examples of completions obtained from Moral Stories dataset,
from OpenAI models of increasing size. Examples were randomly
selected}\label{fig:scale_examples_gpt}
}
\end{figure}
\twocolumn

\end{document}
