% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{subcaption}
\usepackage{graphicx}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

\usepackage{amsmath}
\usepackage{float}
\usepackage{graphicx}
\usepackage{caption}
\usepackage{enumitem}
\usepackage{booktabs}
\usepackage{todonotes}
\usepackage{mathtools}

\makeatletter
\def\blfootnote{\xdef\@thefnmark{}\@footnotetext}
\makeatother
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

% \title{Native Language Prediction and Language Similarity Analysis in Reading
% Patterns of English as Second Language Learners: a Replication Study}
\title{Native Language Prediction from Gaze: a Reproducibility Study}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}


\author{Lina Skerath \\
  IT University of Copenhagen \\
  %\texttt{lske@itu.dk} \\
  \And
  Paulina Toborek \\
  IT University of Copenhagen \\
  %\texttt{pato@itu.dk} 
  \And
  Anita Zielińska \\
    IT University of Copenhagen \\
  %\texttt{anzi@itu.dk} \\
  \AND
Maria Barrett \\
  IT University of Copenhagen \\
  \texttt{mbarrett@itu.dk}\And
  Rob van der Goot \\
  IT University of Copenhagen \\
  \texttt{robv@itu.dk} \\
  }

\begin{document}
\maketitle
\begin{abstract}
% Not only a short version of the paper; this is your sales pitch. Cover the problem/research question, why is it relevant and how did you solve it? Commonly ended with your most important result/finding.
Numerous studies found that the linguistic properties of a person's native language affect the cognitive processing of other languages. However, only one study has shown that it was possible to identify the native language based on eye-tracking records of natural L2 reading using machine learning.  %we use a recently published eye-movement corpus to replicate this finding on a larger and more interrelated set of native languages than in the original study. 
A new corpus allows us to replicate these results on a more interrelated and larger set of native languages.
Our results show that comparable classification performance is maintained despite using less data. However, analysis shows that the correlation between L2 eye movements and native language similarity may be more complex than the original study found.
\end{abstract}

\section{Introduction}
 Research has shown that a speaker's native language can affect their learning and performance in a foreign language~\cite{Berkes2012MultilingualismNP,alonso2016crosslinguistic,cop2017presenting}. The eye movements of a reader, namely fixations and saccades, are a window to the online cognitive processing of text with milliseconds accurateness \cite{rayner1998eye}.
Native speakers of different languages may exhibit different eye movement patterns when reading a foreign language, with those reading in their native language making shorter and more frequent fixations while making longer fixations due to the increased cognitive load when reading in other languages ~\cite{hopp, rayner2012psychology,berzak2022celer}.

Several researchers have examined eye-movement patterns across different nationalities, exploring various aspects such as sentence reading times, fixation count, and saccade duration~\cite{uschi}. \citet{roberts} showed that gaze data could be used for examining, e.g., reading processes, second language acquisition, and discourse processing, as well as give relevant insights into fields of second language acquisition and processing. Early research in Native Language Identification \cite{tsur-rappoport-2007-using} focused on the relationship between a person's native language and their writing in a second language, while \citet{gaze} for the first time predicted a reader's native language using machine learning across four languages (Chinese, Japanese, Portuguese, and Spanish) using only eye-tracking features from natural reading in their second language (L2), English. The study leveraged the knowledge that different languages have unique features, such as word order, grammatical rules, and phonological features, that affect language processing in other languages.

Despite a general interest in eye-tracking corpora for L2 reading, e.g., \cite{cop2017presenting}, until recently, there has not been a publicly available dataset with enough languages to reproduce the results of \citet{gaze}. \citet{gaze} used a subset of the licensed CELER dataset \cite{berzak2022celer} which is the largest eye-tracking corpus by the number of L2 readers encompassing five different native language backgrounds. The Multilingual Eye-movement COrpus (MECO) L2 dataset \cite{meco-l2}\footnote{Publicly available at \url{https://osf.io/q9h43/}} comprises English L2 reading by 12 different language backgrounds and allows replication of the findings by \citet{gaze} on a different and larger set of languages which is why we employ the MECO dataset for this study.


In this study, we replicate the study by \citet{gaze} and classify the native language of the reader from eye-tracking records of them reading English from another corpus.\footnote{The code and data used in the project is publicly available at \url{https://github.com/linaskerath/ANLP_project}} We include readers from seven different language backgrounds that are more interrelated than the original study; the linguistic similarity of the languages used in this study is in the range of 0.64--0.89\footnote{The calculation is explained in §\ref{sec:lingbased_sim}}. The original study did not explore languages in this range but only less similar languages (linguistic similarity <.5) plus one very similar language pair (linguistic similarity >.95).
 
 %our main objective is to analyze the impact of cross-linguistic influence on reading English as a second language (ESL). We investigate the hypothesis that the reading patterns of ESL learners in English text vary depending on their native language due to linguistic factors. To explore the similarities and differences in reading patterns across various native languages, we adopt the methodology outlined in the study by \citet{gaze}, utilizing a different, recent resource, the Multilingual Eye-Movements Corpus (MECO) ~\cite{meco-l2}.

% In this paper, the authors also discuss the hypothesis that native language linguistic factors are paralleled in differences in reading English text. Taking inspiration from their work, we focus on the analysis of cross-linguistic influence in ESL (English as Second Language) reading. We investigate the hypothesis that English text reading pattern differences across native languages are related to linguistic factors. We study similarities in the reading patterns among different native language speakers, on a larger scale than in the original study. 

% We answer the following Research Question:

% \begin{center}
% \textit{Is the preservation of native language similarities paralleled in reading patterns of English as Second Language (ESL) learners?}
% \end{center}

% To answer the research question, we closely follow the experiments of~\citet{gaze}, 
% implementing the same classification model and using described features in the eye-movement analysis. 
% We note, however, that due to the impreciseness of the paper, we can not assure an exact replication of the work but merely as close a reproduction as our interpretation allows. The objectives of our project are to reproduce the following parts of \citet{gaze} research:
% \begin{itemize}
%     \item The training features of the Words in Fixed Context (WFC) feature set for the Shared Sentence Regime.
%     \item The majority class baseline.
%     \item The log-linear model and its performance on the WFC feature set in the Shared Sentences Regime for each of the n-gram sections.
%     \item The figure of linguistic similarities, part (a) and (b) $^{*}$.
% \end{itemize}

% $^{*}$ Since we do not replicate the  Syntactic Clusters (SC) and the  Information Clusters (IC)  feature sets, we can not exactly replicate this last part. We produce this figure using the Words in Fixed Context (WFC) feature set.

% as we want to explore on more lang, this work...
% In this project we explore the aforementioned tasks on more than four languages, thus, this work makes use of a newly created Multilingual Eye-Movements Corpus (MECO) L2 corpus and focuses on exploring similarities between 7 different languages.

%The code to our project is available at \url{https://github.com/linaskerath/ANLP_project}


% \section{Related Work}
% Include at least 2 relevant scientific research paper which provide the basis or motivation for (parts of) your work.



\section{Data}

% The existence of high-quality, comparable data from diverse languages is essential for a comprehensive analysis of the subject. The recently created Multilingual Eye-movement Corpus (MECO) \cite{dataset} meets this premise. 
% The data set\footnote{The data set is publicly available at \url{https://osf.io/q9h43/}} is a product of international collaboration and includes eye-tracking input for speakers of 12 languages recorded during reading an English text in an encyclopedic style. The languages include Dutch, English, Estonian, Finnish, German, Greek, Hebrew, Italian, Norwegian, Russian, Spanish, and Turkish of 543 different participants. 
The MECO data was collected in 12 eye-tracking laboratories around the world. Participants were young adults ranging from 18 to 39 years old with high levels of L2 proficiency, which was ensured through English instruction in higher education.
For more comprehensive information about the dataset, we refer to the authors' paper \cite{meco-l2}.

The MECO data set includes eye-tracking input gathered from native speakers of 12 languages recorded during reading an English encyclopedic text. Due to an insufficient number of participants in some of the cohorts, we used the subset of seven languages with the most participants. To avoid overfitting, we randomly undersampled 23 participants for the two largest cohorts, equivalent in size to the third largest group within the dataset as shown in Table \ref{tab:group-sizes}. \citet{gaze} used 36 to 37 readers for each language.% and the shared regime.

We only use the texts read by all the participants (also named ``shared regime'' in~\citet{gaze}). The total amount of words   read per participant is 595 words, while the original study used 900 words. The feature set employed comprises three word-based measurements: First Fixation duration (FF), First Pass duration (FP) which is the sum of all fixations during the first pass reading of the word, and Total fixation duration (TF).



% \todo{Can one of you add info about the age the genders and English proficiency here (and refer to berzak for more details)?, I wouldnt include distributions, but just ranges as you are not providing the data, just sampling from an existing set. Alternative location=limitations}
% \subsection{Original data set}
% The original study of NLIR was based on a data set including three eye-tracking metrics, gathered for native speakers of five different languages. Measurements collected were:
% \begin{itemize}
%     \item First Fixation duration (FF) - duration of the first fixation on a word.
%     \item First Pass duration (FP) - time spent from first entering a word to first leaving it (including re-fixations within the word).
%     \item Total Fixation duration (TF) - the sum of all fixation times on a word.
% \end{itemize}

% These three features were used in two experiments, with the shared and individual regimes. We followed the shared regime setup in our work, where all participants read the same texts. 
% % The individual regime consisted of data gathered from participants reading different sets of texts and is not included in our work. 
% For a shared regime, the authors of the original study used 78 sentences from the Wall Street Journal part of the Penn Treebank data set.

% The data corresponds to Words in Fixed Context feature set from \citet{gaze} and will therefore be used in the project.

% \subsection{Data exploration}
% The data selected for this study is obtained in a different way from the original research, thus it is necessary to carefully explore it and prepare for further use. The MECO L2 data set, used in this analysis, consists of a variety of metrics on the word level, which corresponds to the needed depth of analysis. However, fixation metrics included in the MECO data set are named differently and we have to select equivalents manually. Therefore, we cannot exclude the possibility that the measured fixation times are slightly different from those in the original study.

% Measures selected for our analysis are:
% \begin{itemize}
%     \item First Fixation duration: firstfix.dur - duration of the first fixation on the word
%     \item First Pass duration: firstrun.dur - time the word was read during first-pass reading
%     \item Total Fixation duration: dur - total time the word was read during the trial in ms (total reading time)
% \end{itemize}

% Moreover, despite the comprehensiveness of the data set used, it is not complete for our analysis requirements. 
% As we follow a shared regime structure, the same texts and sentences need to be read by all participants. Such limitation substantially reduces the number of languages that can be included in the research. As we decided to incorporate in the model only those languages for which the number of participants is greater or equal to eighteen while retaining a set of texts which together would yield a set of approximately 700 words, we managed to include 7 languages in the final data set. In a data set with such limitations, Finnish and Estonian languages appeared to be over-represented, so we decided to randomly subset such that the number of participants was equal to the number of the third-largest group, which was 23. 
%As a final result, we get a data set of 595 words read by 149 participants, with a split between languages shown in Table \ref{tab:group-sizes}.

 \begin{table}[t]
 \centering
 \begin{tabular}{llc}
 \toprule
 \textsc{Language} & ISO & \textsc{$n$ participants} \\ \midrule
 Estonian & et & 23                     \\
 English & en  & 21                     \\ 
 Finnish & fi  & 23                     \\ 
 German & de  & 23                     \\ 
 Hebrew & he  & 18                     \\ 
 Italian & it  & 20                     \\ 
 Spanish & es  & 21                     \\ 
 \bottomrule
 \end{tabular}
 \caption{Number of participants by native language and language ISO code in the data set.}
 \label{tab:group-sizes}
\end{table}

%\begin{table}%[ht]
%\centering
%\setlength\tabcolsep{3pt} % default value: 6pt
%\resizebox{1\columnwidth}{!}{
%\begin{tabular}{llc |llc}
%\toprule
%\textsc{Lang.} & ISO & \textsc{\#part.} & \textsc{Lang.} & ISO & \textsc{\#part.} \\ \midrule
%Estonian & et & 23           & Hebrew & he  & 18       \\
%English & en  & 21            & Italian & it & 20       \\ 
%Finnish & fi  & 23            & Spanish & es & 21        \\ 
%German & de  & 23             &        \\ 
%bottomrule
%\end{tabular}}
%\caption{Number of participants (\# part.) by native language and language ISO code in the data set.}
%\label{tab:group-sizes}
%\end{table}
%\todo{Use actual ISO codes - also in the remaining part of the paper; I think we often use these: \url{https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes}, I have changed them in the table only for now}

%\todo{Table 1: I liked the old table layout better}


% \subsection{Data preparation} \label{sec:data_prep}


\section{Methods} % Methodology
% Here you define and describe your methods, with precise mathematics where applicable. It is highly recommended to add a visualization of your main model. Make sure it is clear what data it is trained on, what features your use. Include a description of your evaluation setup, including a description of which evaluation metrics you chose.
% In this chapter, we describe the methods we used to replicate the original study. As said before, due to the imprecise nature of the original paper, we operated according to our interpretation of the methods used. 
In this section, we describe the methods employed to replicate \citet{gaze}, giving a detailed description of the steps deviating from the setup of the original study.


\subsection{Features} \label{sec:features}
%The MECO data set has already been pre-processed and thus does not require extensive work before using it in a model.
%\todo{what is actually done, do we need above sentence? R: Not really I think}
All data gaps encountered in the MECO dataset related to words marked as skipped by participants during reading, so it is legitimized to replace such shortages with zeros. Additionally, following the approach of the original research, we normalize all fixation times with the reading time of the entire sentence. The final data set consists of three fixation measures columns per word or cluster, where each row represents data collected from one person.
%\todo{Mention that we do n-grmas before introducing feature sets to make it clear that that is consistent throughout all feature sets. R: this actually sounds more confusing to me, as the 3-grams are not the same for all of them.}

\paragraph{Words in Fixed Context (WFC)}
The WFC feature set considers the fixation times for specific words, and no aggregation is performed on the unigram level. The bigrams and trigrams fixation times are then obtained by simply summing values of unigrams that are a part of the interest area.  Columns of the dataset consist of the 3 features for every $n$-gram in the corpus - 5364 features in total. 

\paragraph{Syntactic Clusters (SC)}
%For this setting, we use syntactic labels for grouping the features, for example, the average fixation time on the POS of the word of one participant is a single feature.
In~\citet{gaze}, syntactic features were obtained from the original Penn Treebank. As no manually annotated syntactic features are available for our data we use predicted syntactic information instead (described in detail in Appendix~\ref{app:parser}). Following~\citet{gaze} we use the average FF, FP and TF over n-grams (n=1-3) of the UPOS labels, PTB POS tags, and UD dependency labels as features. For example: the average fixation time of a participant on the UPOS sequence ADV ADJ is a single feature.
%We used MaChAmp v0.4~\cite{van-der-goot-etal-2021-massive} with default settings, trained on the English Web Treebank v2.11 (because it has PTB tags and is English). We evaluated multiple language models and qualitatively found XLM-R Large~\cite{conneau-etal-2020-unsupervised} to perform robustly (UPOS 97.7 accuracy, PTB POS 97.5 accuracy, 91.0 LAS for dependency parsing). More details about the predicted labels can be found in Appendix~\ref{app:parser}.

% XLMR
% UPOS: 0.9769646603195095
% XPOS: 0.9747055026625787
% dep: 0.9095530095207358
% MLUKE is a bit higher

\paragraph{Information Clusters (IC)}
Next to grouping the features by syntactic labels, the average fixation times were calculated for clusters created by the length of the words, measured as a number of characters. For bi- and trigrams, lengths of words were summed and thus clusters were created based on this sum.

%For our last set of features, we use the averages of our three features on words with the same length (in number of characters).  


% \todo{expand feature descriptions!}

% \textit{describe how a feature is created (one n-gram - one feature) bc else we make it as unclear as the original study. Also describe normalization  }

% \todo{Describe training data set, traintest split etc}

% \textit{describe how the final data set looks like: how many columns, features etc}



\subsection{Model}

% To obtain a baseline performance of our data that is comparable with \citet{gaze} results (table \ref{tab:gaze_results}), we compute a majority class benchmark as follows. We assess that the largest class in our data set is shared by several languages. Regardless of which we choose, however, the baseline performance is the same with 15.44$\%$ accuracy. 
For interpretation, we compare to a majority class baseline.
%To obtain a comparable baseline, we calculate the prevalence of the majority class with the most participants. % and recreate the random clusters baseline by randomly assigning words to a cluster where the number of clusters equals the number of syntactic categories in our data. 
Following the original paper, we use a log-linear model to obtain the Native Language Identification from Reading (NLIR) performance as well as the model-based language similarity (\ref{sec:model-based-similarity}).
% \begin{equation*}
%     p(y|x;\theta) = \frac{\exp(\theta \cdot f(x, y))} {\sum_{y^{'} \in Y } \exp(\theta \cdot f(x, y^{'}))}
% \end{equation*}
% where y denotes the native language of the participant, x represents the input features that are described in section \ref{sec:data_prep} and $\theta$ are model parameters.
% We implement the model using python's scikit-learn library. A log-linear model is, in this case, a multinomial logistic regression. We use the `lbfgs' solver in accordance with the original paper.
We implement the model using scikit-learn~\cite{scikit-learn} and use the `lbfgs' solver in accordance with the original paper.
A reader's native language encoded as a categorical variable is used as the model's target variable. 
We report our results based on 10-fold cross-validation.  
To preserve a similar distribution of languages in train and test data, we employ a stratified K-Folds split.
We train the same model on the three feature sets described in the previous section and an additional combination of SC and IC feature sets. 
% Since we only reproduce the Words in Fixed Context approach for the shared regime, we only train the model on the features described in section \ref{sec:data_prep}. 
%In accordance with~\citet{gaze}, we train the model and report accuracy scores for three models: one with unigrams, one with uni+bigrams and one with uni+bi+trigrams.

To ensure comparability with the original paper despite the different amounts of languages, we analyze model performance with different amounts of languages. 
We train the model on each possible combination of languages and group them by the number of languages. %(classes). 
We take the mean accuracy score of each group size and plot the results (figure \ref{fig:comb}). We note that our classes are slightly imbalanced, so arguably F1 could be a better metric but to compare to previous work  and because the classes are almost balanced, we choose to use accuracy.
%Each model was trained on all possible features (uni-, bi-, and trigrams together).




\subsection{Similarity metrics}
% To recreate the plots from the original study showing the relationship between model-derived language similarity originating in second language reading data and the linguistic similarity of languages based on the cosine similarity of their syntactic and geographical features, we first have to derive the similarity metrics.

% The model-based similarity metric serves as an indicator of linguistic properties in Second Language (ESL) learners' reading patterns and the cosine similarity metric is a more objective external comparison between languages.
% We use these features to plot a scatter plot and two comparative language trees.
 \citet{berzak2014reconstructing, gaze} suggest a link between English as a second language (ESL) production and linguistic similarities.
To recreate the language similarity plots from the original study, we derive
the same model-based metric and a cosine similarity based on syntactic and geographical features of a language.


\begin{table}
\centering
\resizebox{0.99\columnwidth}{!}{
\begin{tabular}{l|c|c|c} 
%\hline
& \multicolumn{3}{c}{\textbf{Shared regime}} \\
\hline
 \textbf{Majority Class} &\multicolumn{3}{c}{15.44} \\
 %\textbf{Random Clusters} &\multicolumn{3}{c}{58.38}\\

\hline
 & unigrams & +bigrams & +trigrams \\ %[0.5ex] 
\hline

\textbf{IC} & 47.52 & 48.19 & 48.86 \\
\textbf{SC} & 57.62 & 73.29 & 76.57 \\
\textbf{SC+IC} & 52.29 & 73.29 & 77.95 \\
\hline
%\textbf{WFC} & 65.00 & 67.33 & 66.00 \\
\textbf{WFC} & \textbf{81.29} & 79.29 & 77.95 \\


% \textbf{Words in } \\ \textbf{Fixed Context}\\ \textbf{(WFC)}
% & 65.00 & 67.33 & 66.00 \\ 
\hline
\end{tabular}}
\caption{NLIR results for log-linear model and majority class baseline.}
\label{tab:results}
\end{table}

\subsubsection{Linguistic-based similarity}
% \todo{either make this bold and not sectioned or do features (above) as subsections to make consistent. R: I made the features a subsection, the structure was strange otherwise. These could still be paragraphs, I do not have a strong preference.}
\label{sec:lingbased_sim}
We use the same procedure and data as the original study to derive this similarity metric.The data is obtained from URIEL Typological Compendium \cite{uriel}. Information selected is data derived from the World Atlas of Language Structures, features from Syntactic Structures of the World's Languages, and data from parsing the prose topological descriptions in Ethnologue. This information is supplemented by data on the languages belonging to different families, retrieved from Glottolog's world language tree. We use lang2vec~\cite{littell-etal-2017-uriel} for obtaining the complete feature vectors (with KNN completion). After truncating features with the same values among all languages,\footnote{Note that this can be considered non-standard, as the features of a language might impact the similarity between two other languages. We mainly used this strategy to follow the previous setup} we get a total of 189 features. The similarity scores between languages are then calculated as a cosine similarity of their feature vectors.

\subsubsection{Model-based similarity}\label{sec:model-based-similarity}
% To examine how  native language similarities are paralleled in reading patterns, we obtain a language similarity metric that is based on the aforementioned model.
The model-based similarity captures native language similarities paralleled in reading patterns. 
In the same way as \citet{gaze}, we define ``the classification uncertainty for a pair of native languages $y$ and $y^{'}$ in our data collection $D$, as the average probability assigned by the NLIR classifier to one language given the other being the true
native language.'' It is called  English Reading Similarity (ERS) and is defined as:

{\small
\begin{equation*}
    ERS_{y, y^{'}} = \frac{\sum\limits_{(x,y) \in D_y} p(y^{'}|x; \theta) + \sum\limits_{(x,y^{'}) \in D_y^{'}} p(y|x; \theta)}{|D_y| + |D_y'|} 
\end{equation*}}


%The intuition behind comparing classification uncertainty to linguistic similarity is to evaluate whether similar languages (high similarity) are harder to classify (high uncertainty)

The model, trained on all seven languages to perform NLIR, is used to extract language similarity. We separately feed test data sets for a single language $y$ at a time and extract prediction probabilities for each other language $y^{'}$. Then a mean of the two language probabilities is calculated. 

It is suggested that a higher classification uncertainty indicates greater language similarity.  In figure \ref{fig:scatter} we plot the similarity metrics against each other to test this in the original study implied link. 

% We assume that the inverse of prediction probability is considered to be the model's uncertainty. 

\section{Results}
% Here you provide the technical results of your method over the data. Any tables with numerical results should be put in this section. Use the most meaningful representation of your results that best illustrate the points you want to make. Make sure tables and figure have descriptive captions. If you use figures, make sure they contain labels for each axis.




Table \ref{tab:results} presents the results for the baseline and the log-linear model when using 10-fold cross-validation. The model is trained and evaluated on all seven languages.

%\todo{format after all text is done}




\begin{figure}
    \centering
    %\todo{Add Berzak's result to Figure 1 for reference}
    \includegraphics[width=0.5\textwidth]%[scale=0.8]
    {figures/performance\_graphs\_sns\_all_2.png}
    \caption{Mean performance of all combinations of languages using uni+bi+trigram features.}
    \label{fig:comb}
\end{figure}

% \begin{table}
% \centering
% \resizebox{0.99\columnwidth}{!}{
%     \begin{tabular}{l|rrrr}
%     \toprule
%     & & \multicolumn{3}{c}{N-grams} \\
%     & Majority & 1 & 1-2 & 1-3 \\
%     \midrule
%     WFC & 15.44 & 65.00 & 67.33 & 66.00 \\ 
%     Syntax & \\
%     \bottomrule
%     \end{tabular}}
%     \caption{Results for majority baseline and the different n-gram sizes.}
% \end{table}

% All variants of the model perform substantially better than the majority class baseline. While adding trigrams gave an improvement over using unigrams only, it didn't get as high results as using unigrams and bigrams only. The model trained on these two reached the highest cross-validation accuracy of 67.33\%.

All variants of the model perform substantially better than the majority class baseline.
Similarly to the results by \citet{gaze},
the model trained on the WFC feature set achieves the highest cross-validation accuracy (81.29\%).
While the model trained on syntactic and information cluster features improves with additional bi- and tri-grams, the words in the fixed context feature set do not follow this trend which differs from the original paper's results.  


Since the original study was done with a different number of languages, we investigate how the performance changes  depending on the number of target classes. %We find that, on average, model performance decreases with additional languages, which can be seen in figure \ref{fig:comb}.
Figure \ref{fig:comb} shows the changes in model performance depending on how many target classes it has. E.g., 3 on the $x$ axis corresponds to a group of all combinations ($C_7 ^{3}$) of any three languages in the train set. The y-axis shows the mean performance of all classifiers in that group.
 The results of each classifier in a group vary, thus, we plot the mean performance. As expected, we see that for all feature sets the performance drops when the number of language increase. %The mean performance of the model trained on four languages, which is the number of languages in the original paper, is  

%\vspace{6cm}

\begin{figure}
    \centering
    \includegraphics[scale=0.49]{figures/scatter_10fold.png}
    \caption{Linguistic similarities from URIEL against mean NLIR classification uncertainty of the unigram SC+IC model.} 
    \label{fig:scatter}
\end{figure}




\section{Discussion} % and Error Analysis
% Here you interpret the results back into the setting of the original research question and you dig deeper. What were major findings? What were major shortcomings of your methodology/data?

%\todo{missing discussion}

As evident from Table \ref{tab:results}, our model seems to perform similarly to the original paper's results (Table \ref{tab:gaze_results}, Appendix~\ref{app:berzakResults}). We can not compare these results directly due to the difference in languages, yet, for all combinations of four languages in our data set, we observe in Figure \ref{fig:comb}) that the average performance is 81 $\%$ (compared to 71\% in~\citet{gaze}. However, since we train our model with 3 more languages than the original study and still get similar results, we can confirm that machine learning models can pick up the differences in reading patterns of different native language readers. Contrary to the original paper, we do not see large improvements in performance with additional bigram and trigram features. 

% Figure \ref{fig:comb} was created after we found that the classifier trained on 4 languages performed worse than trained on 7 languages. We later found that this was just an unfortunate combination of languages picked. But to investigate the trend, we trained all language combinations and found, that, on average, model performance decreases with additional languages. In the case we were investigating we had picked Estonian, Finnish, Italian, and Spanish expecting to parallel the original paper's differences in languages, having two pairs of similar languages. With the lower NLIR results and as figure \ref{fig:scatter} shows, however, our model can distinguish between languages but it does not pick up similarities between them. 

% Since the original study was done with a different number of languages, we investigate how the performance changes  depending on the number of target classes. We find that, on average, model performance decreases with additional languages, which can be seen in figure \ref{fig:comb}.

We also explore language similarity by looking at the suggested positive correlation between classification uncertainty and linguistic similarities. Results from~\citet{gaze} are included in Figure~\ref{fig:berzakScatter}, Appendix~\ref{app:berzakResults} for convenience. The plot reproduced in Figure \ref{fig:scatter} does not seem to confirm this hypothesis as no clear trend is visible. We observe that the uncertainty when classifying native speakers vs. L2 reading is substantially lower (mean 0.01) than when distinguishing two groups of L2 readers from those of different native languages (mean 0.11).
We also compute a correlation coefficient of 0.06 which does not indicate a significant correlation found by Berzak et al. 
Similarly, Ward hierarchical clustering for linguistic similarities and classification uncertainty, presented in Figure \ref{fig:ward}, does not present a closeness between grouping using either of these metrics. The plots have little overlaps on the set of languages we used, contrary to the original finding, see Figure \ref{fig:berzakTrees}, and share a little similarity both in terms of languages in each cluster and the general shape of the tree. This suggests that the relation between the English reading patterns and language similarities of the native language found by \citet{gaze} may be more nuanced than the original plot (Figure \ref{fig:berzakTrees}, Appendix~\ref{app:berzakResults}) initially suggests. 


\begin{figure}
\centering
\begin{subfigure}{0.48\linewidth}
    \includegraphics[width=\linewidth]{figures/tree_cosine_gray.png}
    \caption{Linguistic similarities}
    \label{fig:tree1}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
    \includegraphics[width=\linewidth]{figures/tree_LRS_gray.png}
    \caption{Class uncertainty}
    \label{fig:tree2}
\end{subfigure}
\caption{Ward hierarchical clustering. Based on the unigrams SC+IC model. }
\label{fig:ward}
\end{figure}

% The scatter plot is not an exact replication of the original study's results, since we used the WFC feature set and not the SC+IC feature sets to compute model-based similarity. This could be the reason why our plot does not pick up on the trend, suggested by \citet{gaze}. It could also be that the authors made a lucky pick of languages. This, however, needs to be investigated further. \todo{update with new results}


% Following~\citep{gaze}, we plot a ward hierarchical clustering of languages using the similarity metrics (Figure~\ref{tree1}) and~\ref{tree2})
% %The last plot we reproduced in figure \ref{fig:tree1} and \ref{fig:tree2} shows a ward hierarchical clustering of languages using the similarity metrics. 
% Contrary to the previous scatter plot, this plot seems to convey language similarities better. It finds Italian and Spanish to be related languages and the same for German and English. We can not rule out coincidence here, because Estonian and Finnish land very far from each other. It is evident, that our model does not pick up similarities as well as in the original study, be it the wrong training features or difference in languages, we can not tell.

%\subsection{Result analysis}





% \subsection{Future work}

% % In this project we explored only one part of the work done in the original paper. Besides using only three introduced features for unigrams, bigrams and trigrams, with more time additional features described by authors could be created and used for training the model. These include using POS tags and syntactic relation labels as well as clustering the words according to their length.
% In this project, we recreate linguistic similarity metrics but for a different feature set (WFC) than the authors of the original study. By creating the Syntactic and Information clusters by assigning POS tags and measuring word lengths, we could fully recreate the plots exactly and analyse the results better.  This could form a solid base for a future project. Moreover, a possible project continuation could include more exploration of the individual regime, since most practical applications would likely not have the same data for all users. 

% Additionally, more advanced modes than multinomial logistic regression for classification could be experimented with. By that, we could find whether the training data contains more complexity to be learned.

%  Since MECO L2 data set contains more features than the data set described in the original work, there is some prospect in exploring other features and the impact they might have on the task. This step, however, requires expert domain knowledge. 

%\subsection{Limitations}

%Try more complex models?

%Explore the individual regime?

%Reflect on language selection, and its differences compared to Berzak 

% Also, fixing error that we describe would be a great start for a future project.


\section{Conclusion}
% Here you succinctly summarize you work; what do your findings mean for the broader field? what are limitations of your work?

% The aim of our study was to explore the research question asked in \textit{Predicting Native Language from Gaze} \citep{gaze}, namely whether there is a connection between reading patterns in English and the reader's native language. The use of eye movement information may be valuable, but due to the lack of large data sets suitable for NLIR tasks, this is still an unexplored area.

% The problem at hand can be considered as two separate tasks - one of which is checking if it is possible to replicate the model and achieve accuracy close to the original study, and the latter comparing the obtained similarity of languages based on uncertainty to known linguistic knowledge. Despite the satisfactory performance of the trained model, the results of investigating the relationship between reading patterns and linguistic similarity are not so obvious and leave room for discussion and further work. Nevertheless, the outcome of the above study seems to confirm that this is a promising area and, even if it needs some fine-tuning, its investigation is not entirely without merit.
We replicate the finding of \citet{gaze} and are the first to confirm their finding that a reader's native language can be predicted from gaze patterns when reading English text. Having a larger set of more interrelated languages than the original study, we achieve comparable classification results supporting the suggested cross-linguistic influence from the native language to L2.
Despite the satisfactory performance of the NLIR model, the results of investigating the relationship between reading patterns and linguistic similarity are not as straightforward. We believe the relation to be more nuanced than suggested as we are not able to replicate the same outcomes.

\section*{Acknowledgements}
Maria Barrett is supported by a research grant (34437) from VILLUM FONDEN.
% \section{Limitations}

% age, proficiency and gender ranges. 



%\clearpage
\bibliography{bib}
\bibliographystyle{acl_natbib}
% Entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}

\clearpage

\onecolumn
\appendix
\section{Results by Berzak et al. (2017)}
\label{app:berzakResults}
\begin{table}[H]
\begin{center}
\begin{tabular}{l| c |c |c} 
 & \multicolumn{3}{c}{\textbf{Shared regime}} \\
 \hline
  \textbf{Majority Class} &\multicolumn{3}{c}{25.52} \\
    \textbf{Random Clusters} &\multicolumn{3}{c}{22.76} \\
 \hline
  & unigrams & +bigrams & +trigrams \\ %[0.5ex] 
 \hline
 \textbf{Information Clusters (IC)} & 41.38 & 44.14 & 46.21 \\
  \textbf{Syntactic Clusters (SC)} & 45.52 & 57.24 & 58.62 \\ 
 \textbf{Information Clusters (IC)} & 51.72 & 57.24 & 60.0 \\ 
\hline
 \textbf{Words in Fixed Context (WFC)} & 64.14 & 68.28 & \textbf{71.03} \\ 
 \hline
\end{tabular}
 \caption{Native Language Identification from Reading results by \citet{gaze}}
 \label{tab:gaze_results}
\end{center}
\end{table}

\begin{figure}[H]
    \centering
    \includegraphics[scale=0.50]{figures/berzakTrees.png}
    \caption{ Ward hierarchical clustering of linguistic similarities between languages and NLIR average pairwise classification uncertainties by \citet{gaze}} 
    \label{fig:berzakTrees}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[scale=0.5]{figures/scatter_gaze.png}
    \caption{Linguistic similarities against mean NLIR classification uncertainty from \citet{gaze} } 
    \label{fig:berzakScatter}
\end{figure}

\section{Obtaining Syntactic Annotations}
\label{app:parser}
We trained a multi-task MaChAmp model~\cite{van-der-goot-etal-2021-massive}, including UPOS, PTB POS, lemmatization, morphological tagging, and dependency parsing. We used MaChAmp v0.4 with default settings, trained on the English Web Treebank v2.11 (because it has PTB tags and is English). 
It uses the combined (summed cross-entropy) loss of all tasks. We do not use the morphological tags and lemmas but include them for future work. All default hyperparameters are used and the default dev-split is used for model picking.
We first ran the parser on the untokenized input but noticed that it quite commonly outputs the PUNCT label and corresponding relations to (end-of-sentence) words that have punctuation attached. So we pre-split using the BasicTokenizer from huggingface (which only separates punctuations) and use the labels of the words for the combined string.
We compared mBERT~\cite{devlin-etal-2019-bert} with XLM-R Large~\cite{conneau-etal-2020-unsupervised} and MLUKE~\cite{ri-etal-2022-mluke}. We compared their outputs on the MECO dataset manually and found the best performance with the XLM-R Large model (although MLUKE gets higher accuracies on EWT-dev).


\section{Limitations}
The MECO dataset \cite{meco-l2} is recorded at different labs following the same strict protocol. Nevertheless, location and experimenter effects may be confounding factors for the NLIR task. The CELER data \cite{berzak2022celer}, used by \cite{gaze}, seems to all be recorded at the same lab. Since we confirm their hypothesis, we do not see this as a fatal flaw in our study. There is no other available dataset that would allow us to replicate their finding. 


%\todo{Suggested: add a list of the decisions we made, what was the same, what differs from the original paper}


\end{document}
