% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{ACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{hyperref}

% If the title and author information does not fit in the area allocated, uncomment the following
%
\setlength\titlebox{8.25cm}
%
% and set <dim> to something 5cm or larger.

\title{Intriguing Effect of the Correlation Prior on ICD-9 Code Assignment}
%\title{The Negative Impact of Utilizing Correlations in ICD-9 Code Assignment using Language Models}
% alternative

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

% \author[1, 2]{Zihao Yang}
% \author[1, 2]{Chenkang Zhang}
% \author[1, 2]{Muru Wu}
% \author[2, 3]{Xujin Chris Liu}
% \author[1, 2]{Lavender Yao Jiang}
% \author[1, 4, 5]{Kyunghyun Cho}
% \author[2, 6, 7]{Eric Karl Oermann}
% \affil[1]{Center for Data Science, New York University}
% \affil[2]{Department of Neurosurgery, NYU Langone Health}

\author{
  Zihao Yang$^{1,2}$,
  Chenkang Zhang$^{1,2}$,
  Muru Wu$^{1,2}$,
  Xujin Chris Liu$^{2, 3}$, \\
  {\bf Lavender Yao Jiang$^{1, 2}$,}
  {\bf Kyunghyun Cho$^{1, 4, 5, 6}$,}
  {\bf Eric Karl Oermann$^{2, 7, 8, 1}$} \\
  $^{1}$Center for Data Science, New York University \\
  $^{2}$Department of Neurosurgery, NYU Langone Health \\
  $^{3}$Department of Electrical and Computer Engineering, NYU Tandon School of Engineering \\
  $^{4}$Courant Institute of Mathematical Sciences, New York University \\
  $^{5}$Canadian Institute for Advanced Research  \\
  $^{6}$Prescient Design\\
  $^{7}$Department of Radiology, NYU Langone Health \\
  $^{8}$Neuroscience Institute, NYU Langone Health \\
  \texttt{\{gavin.yang,stephen.zhang,wm1077,chris.liu,lyj2002,kyunghyun.cho\}@nyu.edu,} \\
  \texttt{eric.oermann@nyulangone.org}
}

% \author{Zihao Yang Chenkang Zhang Muru Wu \\
%         Center for Data Science, New York University; Department of Neurosurgery, NYU Langone Health}

% \author{Zihao Yang \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\
%   Chenkang Zhang \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Muru Wu \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Xujin Chris Liu \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Lavender Yao Jiang \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Kyunghyun Cho \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Eric Karl Oermann \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And}

\begin{document}
\maketitle
\begin{abstract}
The Ninth Revision of the International Classification of Diseases (ICD-9) is a standardized coding system used to classify health conditions. It is used for billing, tracking individual patient conditions, and for epidemiology. The highly detailed and technical nature of the codes and their associated medical conditions make it difficult for humans to accurately record them. Researchers have explored the use of neural networks, particularly language models, for automated ICD-9 code assignment. However, the imbalanced distribution of ICD-9 codes leads to poor performance. One solution is to use domain knowledge to incorporate a useful prior. This paper evaluates the usefulness of the correlation bias: we hypothesize that correlations between ICD-9 codes and other medical codes could help improve language models' performance. We showed that while the correlation bias worsens the overall performance, the effect on individual class can be negative or positive.\footnote{The implementation code is available on github: \url{https://github.com/nyuolab/text2table}} Performance on classes that are more imbalanced and less correlated with other codes is more sensitive to incorporating the correlation bias. This suggests that while the correlation bias has potential to improve ICD-9 code assignment in certain cases, the applicability criteria need to be more carefully studied. 
\end{abstract}

\section{Introduction}

Electronic Health Records (EHRs) contain patient information in the form of clinical notes, structured data tables, and biomedical imaging and time series. For easy tracking and analysis of health data across different healthcare systems, and critically for billing purposes, hospitals and insurance companies assign codes of a standardized coding system to characterize the clinical conditions of patients. Wrong code assignments may result in billing issues that increase patients' expenses substantially, misdiagnosis, and poor tracking of population level health conditions nationally. The Ninth Revision of the International Classification of Diseases (ICD-9) is a system used worldwide to classify and code diseases, injuries, and other health conditions. There were extensive efforts studying the automated assignment of ICD-9 codes to health records and relevant documents \citep{yan_survey_2022}.

% Although recently updated in ICD-10, a substantial amount of available healthcare datasets continue to use legacy ICD-9 codes as ICD-10 are slowly rolled out.

With recent developments in NLP, there has been a focus on the use of neural networks \citep{yu_automatic_2019, mullenbach_explainable_2018, teng_explainable_2020}. One particularly recent direction is in the use of language models. Originally introduced in BERT \citep{devlin_bert_2019}, the recipe of pretraining and finetuning of language models has shown promising performance in many tasks. Researchers have applied BERT for assigning ICD-9 codes from medical documents \citep{huang_plm-icd_2022, pascual_towards_2021, zhang_bert-xml_2020}. However, BERT and other encoder-based language models perform poorly on ICD-9 code assignment \citep{yan_survey_2022}.

One challenge is the extremely imbalanced distribution of ICD-9 codes. Following the distribution of medical conditions in the real world, some codes occur frequently while other codes may appear only once \citep{yan_survey_2022}. It is difficult for models to correctly predict minority codes because few samples exist in the dataset \citep{sun_classification_2009}. A proposed solution is to incorporate domain knowledge that provides useful priors for the minority codes \citep{bai_improving_2019, wang_automatic_2020, zeng_automatic_2019}.  

We hypothesize that one useful prior for ICD-9 code assignment is the correlation between ICD-9 codes and other relevant coding systems. We term other relevant coding systems auxiliary tasks because language models in our experiments predict codes from these systems in addition to ICD-9 codes. The auxiliary tasks are Current Procedural Terminology (CPT) codes and Diagnosis-Related Group (DRG) codes. This correlation prior stems from the domain knowledge that labels from other coding systems give information about ICD-9 codes. For example, patients who underwent artery bypass surgeries (CPT code 33533) are likely to have heart failures (ICD-9 code 428.0). To test our hypothesis, we investigate the effect of multitasking on correlated auxiliary tasks and encouraging similar label correlations between training labels and model predictions through regularization. We showed that 1) on average, utilizing correlations hurts language models' performance on predicting ICD-9 codes from discharge summaries, 2) for each ICD-9 code, utilizing correlations might hurt or help, 3) ICD-9 codes that are more imbalanced and less correlated with auxiliary tasks experience larger performance changes (both positive and negative) from incorporating the correlation prior. Our findings suggest that the correlation prior has the potential to improve predictions of certain ICD-9 codes, but this method suffers from instability when the main task has an imbalanced label distribution and a weak correlation with auxiliary tasks.


\begin{table*}[h]
\small
    \centering
    \begin{tabular}{llllll}
        \toprule
                   &          &    PROC & PROC+CPT & PROC+DRG & PROC+DIAG \\
        \midrule
        ClinicalBERT & original &  \textbf{0.4528} &    0.397 &   0.3939 &     0.408 \\
                   & CorrLoss &  0.4037 &   0.3594 &   0.3272 &     0.363 \\
        RoBERTa & original &  \textbf{0.4421} &   0.4009 &   0.3884 &    0.4116 \\
                   & CorrLoss &  0.3736 &   0.3236 &   0.2816 &    0.3692 \\
        Longformer & original &  \textbf{0.4712} &   0.4227 &   0.3886 &    0.4219 \\
                   & CorrLoss &  0.4139 &    0.335 &    0.212 &    0.3549 \\
        \bottomrule
    \end{tabular}
    \caption{Macro F1 scores of experiments, in which procedure ICD-9 is the main task, on MIMIC-III-50 test set. For each model, the best F1 score is in bold. PROC means procedure ICD-9. DIAG means diagnosis ICD-9. PROC+CPT means that procedure ICD-9 is the main task and CPT is the auxiliary task. }
    \label{tab:1}
\end{table*}



\section{Related Work}

\paragraph{Domain knowledge} One useful prior for ICD-9 codes is its hierarchical structure. For example, a high-level code (e.g., 428.0 heart failure) encompasses its corresponding low-level codes (e.g., 428.1 left heart failure, 428.2 systolic heart failure). \citet{tsai_leveraging_2019} incorporated this hierarchical prior and improved models' performance on predicting imbalanced ICD-9 codes.  

\paragraph{CorrLoss} CorrLoss is a regularization technique \citep{rieger_corrloss_2022} that encourages consistent label correlations between ground truth and predictions. \citet{rieger_corrloss_2022} uses CorrLoss on the facial affect recognition task to integrate the correlation priors for facial movements. Corrloss can be used in any domain where correlation between prediction targets provides a useful signal. Thus, we adopt Corrloss to integrate information of the correlations between different kinds of diagnosis and procedure codes. 

\section{Methods}


\paragraph{Task overview}
We formulate the task of code assignment into a multilabel text classification task because each patient has multiple codes corresponding to their discharge summaries. Each binary label in the task corresponds to a specific code. Formally, our classifier aims to approximate the probability $p(y_1,\dots, y_n | x)$, where each $y_i$ is an ICD-9 code and $x$ is a discharge summary. 


\paragraph{The Correlation Prior} We hypothesize that correlations between ICD-9 and other coding systems are a useful prior for ICD-9 code assignment and choose to incorporate the prior in two ways.

First, we added the auxiliary tasks of predicting other medical codes (e.g., CPT). Formally, we train a classifier to approximate 
\begin{equation}
    p(y,z|x) = p(y|x)\,p(z|x,y),\label{eq:multitask}
\end{equation}
where $y$ is a sequence of ICD-9 codes (the main task), $z$ is a sequence of other medical codes (the auxiliary task), and $x$ is a discharge summary. Our domain knowledge assumes that the absolute correlation $\mathrm{abs}(\rho(y,z)|x)>0$, so $y,z$ are not conditionally independent given $x$ and $p(z|x,y) \neq p(z|x)$. This is desirable because otherwise, we are strictly increasing the difficulty of the task from learning $p(y|x)$ to learning $p(y|x)\,p(z|x)$.

There are benefits and concerns associated with \autoref{eq:multitask}, and their trade-off is unclear \textit{a priori}. One benefit is that extra dependency information from $p(z|x,y)$ could potentially simplify learning $p(y,z|x)$. One drawback is that the additional prediction targets $z$ could worsen the curse of dimensionality. Whether the benefit would outweigh the drawback is difficult to determine without running a controlled experiment.

Second, we used CorrLoss to encourage similar label correlation patterns between training and predictions. Formally, we added a regularization term $c=\sum_{i\neq j}c(d_i,d_j)$. Each summation term scales with a correlation difference:
\begin{equation}
    c(d_i,d_j) \propto |\rho(d_i,d_j)_{y_{\mathrm{train}}} - \rho(d_i,d_j)_{\hat{y}}|, \label{eq:corrloss}
\end{equation}

where $d_i,d_j$ are different classes, $\rho(d_i,d_j)_{v}$ is the correlation between class $d_i$ and $d_j$ in a vector $v$, $y_\mathrm{train}$ is the training labels, $\hat{y}$ is the predicted labels, and $\rho$ is the Pearson correlation function.


\paragraph{Dataset}
We built two datasets from the Medical Information Mart for Intensive Care III (MIMIC-III) \citep{johnson_mimic-iii_2016}, a database of EHRs. Our first dataset, subsequently referred to as ``MIMIC-III", contains examples of each patient's discharge summary, and associated diagnosis and procedure codes (diagnosis ICD-9, procedure ICD-9, CPT, and DRG). Because this dataset is extremely imbalanced, we further select the top 50 most frequently used codes for each kind of coding system to construct a second dataset that represents a more ideal scenario. Following the convention of related literature, we call this dataset ``MIMIC-III-50" \citep{vu_label_2020, luo_fusion_2021, li_icd_2020}. Statistics of the MIMIC-III dataset are in Appendix \ref{app:A}. 

\paragraph{Models and Evaluation}
We use ClinicalBERT \citep{alsentzer_publicly_2019}, RoBERTa \citep{liu_roberta_2019}, Longformer \citep{beltagy_longformer_2020} (justification in Appendix \ref{app:C}). We use the macro F1 as our metric for comparison because this metric treats all classes equally, which means minority codes are as important as majority codes in evaluation \citep{branco_survey_2016, sun_classification_2009, ferri_experimental_2009}. Because it is an imbalanced classification, the default threshold of 0.5 is not suitable \citep{zhou_training_2006, zou_finding_2016}. Instead, we tune the threshold according to the precision-recall curve to maximize the F1 score for each individual label. 

% We also report the macro precision scores and the macro recall scores of the results but they are not used for comparison and are included in Appendix. 

% \paragraph{Threshold tuning}




\section{Experiments}

To test whether the correlation prior is useful for ICD code assignment, we incorporate multitasking (\autoref{eq:multitask}) and CorrLoss (\autoref{eq:corrloss}) into our model and check if they improve performance. Specifically, we studied two main tasks (diagnosis ICD-9 codes and procedure ICD-9 codes). For each main task, we added one of the three auxiliary tasks: DRG codes, CPT codes, and the other ICD-9 codes (for diagnosis ICD-9 code, the auxiliary task can be procedure ICD-9 code, and vice versa). We trained both main-task-only models and multitasking models with and without CorrLoss. 
% We have two experiments: multitasking and incorporation of CorrLoss. Multitasking means that models predict not only ICD-9 codes but also codes from other coding systems as auxiliary tasks. Through multitasking experiments, we show how providing correlation information affects models.

% In our second experiment, we add CorrLoss, which encourages models to explicitly learn the correlations between labels. Through this experiment, we show how deliberately injecting corelational information into models affects their performance.



% \paragraph{Multitasking} We compare the 

% We studied two dimain tasks ($y$): , and two auxiliary tasks ($z$):  In addition, if the main task is diagnosis/procedure ICD-9 codes, procedure/diagnosis ICD-9 codes can be the auxiliary task. Our baseline is learning the main tasks only (learning $p(y|x)$): diagnosis ICD-9 or procedure ICD-9 (baselines). Then, we add one of auxiliary tasks each time and learn. We evaluate how the auxiliary task affects the performance on the main tasks.

% \paragraph{Corrloss} We modify loss functions of all the models to incorporate CorrLoss and repeat the experiments described above to see the impact of adding auxiliary tasks with and without CorrLoss. Our implementation of CorrLoss follows \citeposs{rieger_corrloss_2022} description.

\section{Results}




\textbf{Multitasking and CorrLoss hurt performance on MIMIC-III-50 and do not significantly impact performance on MIMIC-III.} \autoref{tab:1} shows the macro-F1 score on procedure ICD-9 of the MIMIC-III-50 dataset. We observe two patterns for each language model. First, adding auxiliary tasks always decreases the performance of models in comparison to predicting main tasks only. Second, regularizing with CorrLoss always decreases the performance of models in comparison to not using CorrLoss. The same pattern exists for predicting diagnosis ICD-9 of the MIMIC-III-50 dataset (Appendix Table \ref{tab:6}). However, on the full MIMIC-III dataset, multitasking and CorrLoss do not impact models' performance significantly (Appendix \ref{app:B}). 

\section{Analysis}

\begin{figure*}[bth]
    \centering
    \includegraphics[scale=0.25, trim={0.5cm 1.5cm 0 0.6cm},clip]{performance_change.png}
    \caption{The plot of ClinicalBERT's performance changes (Y axis) on labels of procedure ICD-9, when DRG is added as the auxiliary task, versus the balances (X axis) of the labels, and versus the correlations (sizes and colors of the units) between each label with the whole auxiliary DRG task. CorrLoss is not included.}
    \label{fig:1}
\end{figure*}

Since the macro F1 score does not show significant changes from multitasking and CorrLoss on the full MIMIC-III dataset, we investigate whether the performance changes for individual labels. Specifically, we analyzed how label imbalance (measured by Shannon entropy, defined in Appendix \ref{app:D1}) and label correlation (measured by the average absolute Pearson correlation coefficient between each main task label and all auxiliary task labels, as defined in Appendix \ref{app:D1}) affect the model's performance. 




\textbf{For individual ICD-9 code, incorporating the correlation prior may hurt or help.} \autoref{fig:1} shows that there exist labels with both negative and positive performance changes. 

% \subsection{Performance on Each Label}
\textbf{Labels that are more imbalanced and less correlated to auxiliary labels experience larger changes.} \autoref{fig:1} shows two relationships: (1) more balanced labels (closer to the right) have less performance changes (spread of dots on the y axis), (2) labels that are more correlated with the auxiliary task (darker dots) have less performance changes (spread along the y axis). All the other plots of different tasks and setups show similar patterns (Appendix \ref{app:D1}). 

% The plots do not show that in which scenario utilizing correlations help or harm performance.
\begin{table}[h]
\small
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.333 &    0.273 \\
                   & +DRG &   0.28 &    0.413 \\
                   & +DIAG &    0.3 &    0.387 \\
        RoBERTa & +CPT &    0.4 &      0.3 \\
                   & +DRG &  0.393 &    0.353 \\
                   & +DIAG &  0.313 &    0.287 \\
        Longformer & +CPT &   0.34 &    0.427 \\
                   & +DRG &   0.34 &     0.28 \\
                   & +DIAG &  0.347 &    0.307 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 most balanced procedure ICD-9 labels and on the bottom 50 least balanced procedure ICD-9 labels, with different auxiliary tasks and models. CorrLoss is not included.}
    \label{tab:2}
\end{table}

\begin{table}[h]
\small
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.333 &    0.327 \\
                   & +DRG &   0.32 &    0.327 \\
                   & +DIAG &  0.293 &    0.247 \\
        RoBERTa & +CPT &  0.487 &    0.333 \\
                   & +DRG &  0.373 &    0.387 \\
                   & +DIAG &  0.267 &    0.293 \\
        Longformer & +CPT &  0.433 &    0.327 \\
                   & +DRG &   0.28 &    0.273 \\
                   & +DIAG &  0.333 &     0.24 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 procedure ICD-9 labels that are most correlated with the auxiliary task and on the bottom 50 procedure ICD-9 labels that are least correlated with the auxiliary task, with different auxiliary tasks and models. CorrLoss is included.}
    \label{tab:3}
\end{table}

% \subsection{Impact in Ideal and Extreme Scenarios}

In both extreme scenarios (imbalanced label, small correlation with auxiliary labels) and ideal scenarios (balanced labels, high correlation with auxiliary labels), \textbf{incorporating correlation is more likely to hurt than help.} \autoref{tab:2} shows that for the top 50 most balanced labels and the bottom 50 least balanced labels, if we utilize correlations (with multitasking and CorrLoss), the percentage of positive F1 score changes is always less than 50\%. \autoref{tab:3} shows that for the top 50 labels that are most correlated with the auxiliary tasks and the bottom 50 labels that are least correlated with the auxiliary tasks, utilizing correlations also leads to $<50\%$ positive F1 score change.  

% \paragraph{performance change vs. balance}For every experiment of both main tasks on full MIMIC-III dataset, we examine the top 50 most balanced labels and the bottom 50 least balanced labels. The top 50 labels represents the scenario of balanced dataset and the bottom 50 labels represents the scenario of imbalanced dataset. To see whether utilizing correlations help or harm performance in these scenarios, we calculate the percentages of positive f1 score changes. Table \ref{tab:2} shows the percentages of positive f1 score changes in the scenarios.

% \paragraph{performance change vs. correlation}In addition, we examine the top 50 labels that are most correlated with the auxiliary task and the bottom 50 labels that are least correlated with the auxiliary task. The top 50 labels represents the scenario of high correlations and the bottom 50 labels represents the scenario of low correlations. Table \ref{tab:3} shows the percentages of positive f1 score changes in the correlation scenarios. 

% In both tables, all the percentages are below 0.5. Models' performance improves on less than half of labels in all scenarios we examine. Scenario analyses for different main tasks in different setups have similar outcomes, which are not shown here for brevity (Tables in Appendix \ref{app:D2}). 

\section{Discussion}
% \textbf{Utilizing correlations with language models for predicting ICD-9 codes from discharge summaries leads to decreased performance.} 
Since multitasking and CorrLoss worsen language models' overall performance, it contradicts our hypothesis that the correlations between ICD-9 codes and other medical codes would be a useful prior. Nevertheless, the performance changes on individual labels are more nuanced and show potential for improving prediction of certain ICD-9 codes. We wonder what characterizes the labels that benefit from incorporating the correlation prior (dots with positive changes in \autoref{fig:1}). Perhaps for those labels, the additional dependency information gained from the auxiliary tasks outweigh the increased learning complexity from a larger output space. A prerequisite for a rigorous investigation would be quantifying the trade-off between the dependency information and the learning complexity.

% , a passive approach and an active approach to utilize correlations, worsen language models' overall performances on predicting ICD-9 codes from patients' discharge summaries in a less imbalanced setting (MIMIC-III-50). The analysis of the real-world situation (full MIMIC-III) shows that the impact of utilizing correlations on model performance increases as ICD-9 codes' correlations with added auxiliary codes and their balance decrease. Furthermore, the decrease in model performance on majority of codes, with multitasking and incorporation of CorrLoss in nearly all extreme and ideal scenarios, suggests the methods may be unfavorable ways to incorporate domain knowledge to tackle the issue of imbalanced data. 

We recognize three limitations that may influence the interpretation of our results and call for future works. First, we did not conduct a hyperparameter search for the regularization strength of CorrLoss. Second, since F1 score decreases are substantial and universal across all experiments on MIMIC-III-50, we did not run experiments multiple times with different seeds. Third, we did not provide a rigorous explanation of what caused our empirical findings. Future works can investigate the plausible hypothesis that the trade-off between the dependency information and the learning complexity causes these findings. Besides these limitations, future works can also investigate more scenarios and methods of incorporating the correlation prior.

%Although we lack experimental evidence to explain what caused our findings, we propose a plausible hypothesis that accounts for these findings for future work to investigate. Explained as the benefit and the drawback associated with multitasking and \autoref{eq:multitask}, the trade-off between dependency information and the label space may exists. The drawback of a larger label space as a result of adding auxiliary tasks may outweigh the benefit of extra dependency information, which may cause language model's worse overall performance. To test this hypothesis, future work can quantify the trade-off and design experiments to reveal the relationship between the trade-off and language model's performance. 
%Second, the medical code distribution in the training set may be inconsistent with the distribution in the test set, which may cause that the correlation prior learned by language models from the training set cannot be generalized to the test set. To test this hypothesis, future work can 
%A balance between mutual information and the label space exists in multitasking. Adding auxiliary codes may increase the mutual information provided to a model, which may help the model performance; however, it also increases the size of the label space at the same time, which may hurt the model performance. Future work can explore more scenarios since the benefit of more mutual information may outweigh the disadvantage of a larger label space in certain scenarios. Future work can also focus on other methods that do not involve this balance like CorrLoss which does not bring additional mutual information but emphasize the existing mutual information. It would be beneficial to conduct an extensive hyperparameter search for CorrLoss. In addition, it could be worthwhile to find a method to incorporate additional mutual information without the drawback of a larger label space. 

% Entries for the entire Anthology, followed by custom entries

\bibliography{text2data}
\bibliographystyle{acl_natbib}

\clearpage

\appendix

\begin{figure*}

\section{Dataset Statistics} \label{app:A}
\vspace{0.5cm}
    \centering
    \includegraphics[scale=0.15]{token.png}
    \caption{The distribution of lengths of tokenized discharge summaries in MIMIC-III dataset.}
    \label{fig:my_label}
\vspace{0.5cm}
    \centering
    \includegraphics[scale=0.15]{dicd9.png}
    \caption{The distribution of diagnosis ICD-9. There are 6918 diagnosis ICD-9 codes. 6062 Codes occur less than or equal to 100 times in MIMIC-III dataset. They are not included for clarity.}
    \label{fig:my_label}
\vspace{0.5cm}
    \centering
    \includegraphics[scale=0.15]{picd9.png}
    \caption{The distribution of procedure ICD-9. There are 2011 procedure ICD-9 codes. 1767 Codes occur less than or equal to 100 times in MIMIC-III dataset. They are not included for clarity.}
    \label{fig:my_label}

\end{figure*}


\begin{table*}[t!]
\section{Results} \label{app:B}
\vspace{1cm}
    \centering
    \begin{tabular}{llllll}
        \toprule
                   &          &    PROC & PROC+CPT & PROC+DRG & PROC+DIAG \\
        \midrule
        ClinicalBERT & original &  0.0098 &   0.0094 &   0.0091 &    0.0097 \\
                   & CorrLoss &  0.0102 &   0.0099 &   0.0088 &    0.0087 \\
        RoBERTa & original &  0.0097 &   0.0089 &   0.0087 &    0.0088 \\
                   & CorrLoss &  0.0095 &   0.0095 &   0.0098 &    0.0089 \\
        Longformer & original &  0.0088 &   0.0088 &   0.0095 &    0.0085 \\
                   & CorrLoss &  0.0094 &   0.0085 &   0.0091 &    0.0078 \\
        \bottomrule
    \end{tabular}
    \caption{Macro F1 scores of experiments, in which procedure ICD-9 is the main task, on full MIMIC-III test set.}
    \label{tab:my_label}
\vspace{1cm}   
    \centering
    \begin{tabular}{llllll}
        \toprule
                   &          &    DIAG & DIAG+CPT & DIAG+DRG & DIAG+PROC \\
        \midrule
        ClinicalBERT & original &  0.0068 &   0.0066 &   0.0066 &    0.0067 \\
                   & CorrLoss &  0.0066 &   0.0069 &   0.0069 &    0.0068 \\
        RoBERTa & original &  0.0069 &   0.0065 &   0.0062 &    0.0065 \\
                   & CorrLoss &  0.0071 &   0.0071 &   0.0066 &    0.0065 \\
        Longformer & original &  0.0072 &   0.0069 &    0.007 &    0.0071 \\
                   & CorrLoss &   0.007 &   0.0068 &   0.0076 &    0.0071 \\
        \bottomrule
    \end{tabular}
    \caption{Macro F1 scores of experiments, in which diagnosis ICD-9 is the main task, on full MIMIC-III test set.}
    \label{tab:my_label}
\vspace{1cm}   
    \centering
    \begin{tabular}{llllll}
        \toprule
                   &          &    DIAG & DIAG+CPT & DIAG+DRG & DIAG+PROC \\
        \midrule
        ClinicalBERT & original &  0.3755 &   0.3296 &   0.3351 &    0.3351 \\
                   & CorrLoss &  0.3235 &   0.2966 &   0.2947 &    0.2992 \\
        RoBERTa & original &  0.3851 &   0.3255 &   0.3307 &    0.3341 \\
                   & CorrLoss &  0.3143 &   0.2822 &   0.2713 &    0.2939 \\
        Longformer & original &  0.4408 &    0.349 &   0.3544 &    0.3552 \\
                   & CorrLoss &  0.3364 &   0.2963 &   0.2906 &    0.3027 \\
        \bottomrule
    \end{tabular}
    \caption{Macro F1 scores of experiments, in which diagnosis ICD-9 is the main task, on MIMIC-III-50 test set.}
    \label{tab:6}
\end{table*}

\clearpage


\section{Justifcation of Models} \label{app:C}
The variant of ClinicalBERT we use is Bio+Discharge Summary BERT model because it was further trained on discharge summaries from MIMIC-III after initialized from BioBERT \citep{lee_biobert_2020}.

We use RoBERTa because it is a variant of vanilla BERT that was trained differently to improve its performance on a range of NLP tasks.

We use Longformer because it can handle long text sequences. BERT and many BERT-based models cannnot handle text sequences longer than 512 tokens. Many tokenized discharge summaries are text sequences longer than 512 tokens and Longformer can benefit from more complete understandings of discharge summaries. 

Each model represents a different improvement on top of vanilla BERT: ClinicalBERT improves through domain-specific pretraining; RoBERTa improves through tuning training setup; and Longformer improves through incorporating more information from the input. With these models, we cover a significant part of the improvement spectrum, which shows that the pattern we present is generalizable to different models.

\section{Analysis}
\subsection{Performance on Each Label} \label{app:D1}

\paragraph{Other figures}
Since there are 72 experiments that have auxiliary tasks, there are 72 corresponding plots. Thus, it is unreasonable to include all of them in the appendix. You can find all plots in our github repository: \url{https://github.com/nyuolab/text2table/tree/main/notebooks}.

\paragraph{Shannon Entropy} 

\begin{equation}
    H(X) = -\sum_{i=1}^n p(x_i) \log_2 p(x_i)
\end{equation}


In this equation, $H(X)$ represents the entropy of a label $X$ with possible outcomes $x_1$, $x_2$, ..., $x_n$. In our context, $n=2$ because a label only has two possible outcomes: 1 (positive) or 0 (negative). The term $p(x_i)$ represents the probability of the i-th outcome, and the logarithm is taken with base 2 to give the result in units of bits. The sum is taken over all possible outcomes of $X$. With only two possible outcomes, a label's Shannon entropy will be close to 1 if it is balanced, and will be close to 0 if it is imblanced. 

\paragraph{Representation of Correlations}

\begin{equation}
    C(a, B) = \frac{\sum_{b\in B}|P(a,b)|}{card(B)}
\end{equation}

In this equation, $C(a, B)$ represents the correlations between a label of the main task $a$ and a set containing labels of the auxiliary task. For each label of the auxiliary task $b \in B$, $|P(a, b)|$ represents the absolute value of the Pearson correlation coefficient bettwen $a$ and $b$. $card(B)$ is the cardinality of $B$ (i.e. the number of labels in B).


\subsection{Performance in Different Scenarios} \label{app:D2}

\begin{table}[h]
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.453 &     0.32 \\
                   & +DRG &   0.54 &    0.293 \\
                   & +PROC &   0.48 &     0.38 \\
        RoBERTa & +CPT &   0.48 &    0.313 \\
                   & +DRG &  0.507 &    0.307 \\
                   & +PROC &   0.48 &    0.333 \\
        Longformer & +CPT &    0.5 &     0.32 \\
                   & +DRG &   0.48 &    0.393 \\
                   & +PROC &  0.433 &    0.287 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 most balanced diagnosis ICD-9 labels and on the bottom 50 least balanced diagnosis ICD-9 labels, with different auxiliary tasks and models. CorrLoss is not included in all experiments we examine in this table.}
    \label{tab:my_label}
\end{table}

\begin{table}[h]
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.347 &     0.36 \\
                   & +DRG &  0.327 &    0.313 \\
                   & +DIAG &  0.273 &     0.28 \\
        RoBERTa & +CPT &   0.32 &     0.32 \\
                   & +DRG &  0.353 &     0.36 \\
                   & +DIAG &  0.273 &     0.22 \\
        Longformer & +CPT &  0.353 &    0.367 \\
                   & +DRG &   0.28 &    0.293 \\
                   & +DIAG &  0.307 &     0.26 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 most balanced procedure ICD-9 labels and on the bottom 50 least balanced procedure ICD-9 labels, with different auxiliary tasks and models. CorrLoss is included in all experiments we examine in this table.}
    \label{tab:my_label}
\end{table}

\begin{table}[h]
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.413 &    0.307 \\
                   & +DRG &  0.533 &     0.28 \\
                   & +PROC &  0.487 &    0.293 \\
        RoBERTa & +CPT &   0.46 &      0.3 \\
                   & +DRG &  0.493 &    0.373 \\
                   & +PROC &  0.473 &     0.34 \\
        Longformer & +CPT &  0.453 &    0.293 \\
                   & +DRG &  0.487 &     0.34 \\
                   & +PROC &    0.5 &    0.307 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 most balanced diagnosis ICD-9 labels and on the bottom 50 least balanced diagnosis ICD-9 labels, with different auxiliary tasks and models. CorrLoss is included in all experiments we examine in this table.}
    \label{tab:my_label}
\end{table}

\begin{table}[h]
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.467 &     0.32 \\
                   & +DRG &  0.307 &    0.373 \\
                   & +DIAG &  0.367 &    0.287 \\
        RoBERTa & +CPT &  0.387 &    0.267 \\
                   & +DRG &  0.413 &    0.407 \\
                   & +DIAG &   0.32 &    0.307 \\
        Longformer & +CPT &  0.427 &    0.367 \\
                   & +DRG &   0.34 &    0.307 \\
                   & +DIAG &   0.42 &    0.307 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 procedure ICD-9 labels that are most correlated with the auxiliary task and on the bottom 50 procedure ICD-9 labels that are least correlated with the auxiliary task, with different auxiliary tasks and models. CorrLoss is not included in all experiments we examine in this table.}
    \label{tab:my_label}
\end{table}

\begin{table}[h]
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.507 &    0.333 \\
                   & +DRG &  0.493 &    0.287 \\
                   & +PROC &  0.473 &    0.347 \\
        RoBERTa & +CPT &   0.48 &    0.247 \\
                   & +DRG &  0.513 &     0.36 \\
                   & +PROC &   0.46 &    0.347 \\
        Longformer & +CPT &  0.487 &    0.313 \\
                   & +DRG &  0.493 &     0.34 \\
                   & +PROC &  0.427 &    0.313 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 diagnosis ICD-9 labels that are most correlated with the auxiliary task and on the bottom 50 diagnosis ICD-9 labels that are least correlated with the auxiliary task, with different auxiliary tasks and models. CorrLoss is not included in all experiments we examine in this table.}
    \label{tab:my_label}
\end{table}

\begin{table}[h]
    \centering
    \begin{tabular}{llll}
        \toprule
                   &       &  top50 & bottom50 \\
        \midrule
        ClinicalBERT & +CPT &  0.467 &    0.373 \\
                   & +DRG &   0.52 &      0.3 \\
                   & +PROC &   0.46 &    0.333 \\
        RoBERTa & +CPT &  0.493 &     0.32 \\
                   & +DRG &   0.52 &    0.433 \\
                   & +PROC &  0.473 &    0.253 \\
        Longformer & +CPT &   0.46 &     0.32 \\
                   & +DRG &  0.513 &    0.467 \\
                   & +PROC &  0.453 &     0.34 \\
        \bottomrule
    \end{tabular}
    \caption{The percentages of positive macro F1 score changes on the top 50 diagnosis ICD-9 labels that are most correlated with the auxiliary task and on the bottom 50 diagnosis ICD-9 labels that are least correlated with the auxiliary task, with different auxiliary tasks and models. CorrLoss is included in all experiments we examine in this table.}
    \label{tab:my_label}
\end{table}

\end{document}
