% INTRODUCTION ---------------------------------------------------------------

@article{WU2022364,
title = {A survey of human-in-the-loop for machine learning},
journal = {Future Generation Computer Systems},
volume = {135},
pages = {364-381},
year = {2022},
issn = {0167-739X},
doi = {https://doi.org/10.1016/j.future.2022.05.014},
url = {https://www.sciencedirect.com/science/article/pii/S0167739X22001790},
author = {Xingjiao Wu and Luwei Xiao and Yixuan Sun and Junhang Zhang and Tianlong Ma and Liang He},
keywords = {Human-in-the-loop, Machine learning, Deep learning, Data processing, Computer vision, Natural language processing},
abstract = {Machine learning has become the state-of-the-art technique for many tasks including computer vision, natural language processing, speech processing tasks, etc. However, the unique challenges posed by machine learning suggest that incorporating user knowledge into the system can be beneficial. The purpose of integrating human domain knowledge is also to promote the automation of machine learning. Human-in-the-loop is an area that we see as increasingly important in future research due to the knowledge learned by machine learning cannot win human domain knowledge. Human-in-the-loop aims to train an accurate prediction model with minimum cost by integrating human knowledge and experience. Humans can provide training data for machine learning applications and directly accomplish tasks that are hard for computers in the pipeline with the help of machine-based approaches. In this paper, we survey existing works on human-in-the-loop from a data perspective and classify them into three categories with a progressive relationship: (1) the work of improving model performance from data processing, (2) the work of improving model performance through interventional model training, and (3) the design of the system independent human-in-the-loop. Using the above categorization, we summarize the major approaches in the field; along with their technical strengths/weaknesses, we have a simple classification and discussion in natural language processing, computer vision, and others. Besides, we provide some open challenges and opportunities. This survey intends to provide a high-level summarization for human-in-the-loop and to motivate interested readers to consider approaches for designing effective human-in-the-loop solutions.}
}

@article{DBLP:journals/corr/AmodeiOSCSM16,
  author       = {Dario Amodei and
                  Chris Olah and
                  Jacob Steinhardt and
                  Paul F. Christiano and
                  John Schulman and
                  Dan Man{\'{e}}},
  title        = {Concrete Problems in {AI} Safety},
  journal      = {CoRR},
  volume       = {abs/1606.06565},
  year         = {2016},
  url          = {http://arxiv.org/abs/1606.06565},
  eprinttype    = {arXiv},
  eprint       = {1606.06565},
  timestamp    = {Mon, 13 Aug 2018 16:48:59 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/AmodeiOSCSM16.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@book{doi/10.2759/346720,
author = {European Commission and Directorate-General for Communications Networks, Content and Technology},
title = {Ethics guidelines for trustworthy AI},
publisher = {Publications Office},
year = {2019},
doi = {doi/10.2759/346720}
}

@inproceedings{parketal2022efficient,
    title = "Efficient Classification of Long Documents Using Transformers",
    author = "Park, Hyunji  and
      Vyas, Yogarshi  and
      Shah, Kashif",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.acl-short.79",
    doi = "10.18653/v1/2022.acl-short.79",
    pages = "702--709",
    abstract = "Several methods have been proposed for classifying long textual documents using Transformers. However, there is a lack of consensus on a benchmark to enable a fair comparison among different approaches. In this paper, we provide a comprehensive evaluation of the relative efficacy measured against various baselines and diverse datasets {---} both in terms of accuracy as well as time and space overheads. Our datasets cover binary, multi-class, and multi-label classification tasks and represent various ways information is organized in a long text (e.g. information that is critical to making the classification decision is at the beginning or towards the end of the document). Our results show that more complex models often fail to outperform simple baselines and yield inconsistent performance across datasets. These findings emphasize the need for future studies to consider comprehensive baselines and datasets that better represent the task of long document classification to develop robust models."
}

@inproceedings{10.5555/3495724.3495759,
author = {Teney, Damien and Kafle, Kushal and Shrestha, Robik and Abbasnejad, Ehsan and Kanan, Christopher and van den Hengel, Anton},
title = {On the Value of Out-of-Distribution Testing: An Example of Goodhart's Law},
year = {2020},
isbn = {9781713829546},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
abstract = {Out-of-distribution (OOD) testing is increasingly popular for evaluating a machine learning system's ability to generalize beyond the biases of a training set. OOD benchmarks are designed to present a different joint distribution of data and labels between training and test time. VQA-CP has become the standard OOD benchmark for visual question answering, but we discovered three troubling practices in its current use. First, most published methods rely on explicit knowledge of the construction of the OOD splits. They often rely on "inverting" the distribution of labels, e.g. answering mostly "yes" when the common training answer is "no". Second, the OOD test set is used for model selection. Third, a model's in-domain performance is assessed after retraining it on in-domain splits (VQA v2) that exhibit a more balanced distribution of labels. These three practices defeat the objective of evaluating generalization, and put into question the value of methods specifically designed for this dataset. We show that embarrassingly-simple methods, including one that generates answers at random, surpass the state of the art on some question types. We provide short- and long-term solutions to avoid these pitfalls and realize the benefits of OOD evaluation.},
booktitle = {Proceedings of the 34th International Conference on Neural Information Processing Systems},
articleno = {35},
numpages = {11},
location = {Vancouver, BC, Canada},
series = {NIPS'20}
}

@inproceedings{NEURIPS2022_7a969c30,
 author = {Schrouff, Jessica and Harris, Natalie and Koyejo, Sanmi and Alabdulmohsin, Ibrahim M and Schnider, Eva and Opsahl-Ong, Krista and Brown, Alexander and Roy, Subhrajit and Mincu, Diana and Chen, Christina and Dieng, Awa and Liu, Yuan and Natarajan, Vivek and Karthikesalingam, Alan and Heller, Katherine A and Chiappa, Silvia and D\textquotesingle Amour, Alexander},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {19304--19318},
 publisher = {Curran Associates, Inc.},
 title = {Diagnosing failures of fairness transfer across distribution shift in real-world medical settings},
 url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/7a969c30dc7e74d4e891c8ffb217cf79-Paper-Conference.pdf},
 volume = {35},
 year = {2022}
}

@article{10.1007/s10462-022-10246-w,
author = {Mosqueira-Rey, Eduardo and Hern\'{a}ndez-Pereira, Elena and Alonso-R\'{\i}os, David and Bobes-Bascar\'{a}n, Jos\'{e} and Fern\'{a}ndez-Leal, \'{A}ngel},
title = {Human-in-the-Loop Machine Learning: A State of the Art},
year = {2022},
issue_date = {Apr 2023},
publisher = {Kluwer Academic Publishers},
address = {USA},
volume = {56},
number = {4},
issn = {0269-2821},
url = {https://doi.org/10.1007/s10462-022-10246-w},
doi = {10.1007/s10462-022-10246-w},
abstract = {Researchers are defining new types of interactions between humans and machine learning algorithms generically called human-in-the-loop machine learning. Depending on who is in control of the learning process, we can identify: active learning, in which the system remains in control; interactive machine learning, in which there is a closer interaction between users and learning systems; and machine teaching, where human domain experts have control over the learning process. Aside from control, humans can also be involved in the learning process in other ways. In curriculum learning human domain experts try to impose some structure on the examples presented to improve the learning; in explainable AI the focus is on the ability of the model to explain to humans why a given solution was chosen. This collaboration between AI models and humans should not be limited only to the learning process; if we go further, we can see other terms that arise such as Usable and Useful AI. In this paper we review the state of the art of the techniques involved in the new forms of relationship between humans and ML algorithms. Our contribution is not merely listing the different approaches, but to provide definitions clarifying confusing, varied and sometimes contradictory terms; to elucidate and determine the boundaries between the different methods; and to correlate all the techniques searching for the connections and influences between them.},
journal = {Artif. Intell. Rev.},
month = {aug},
pages = {3005–3054},
numpages = {50},
keywords = {Explainable AI, Active learning, Interactive machine learning, Machine teaching, Curriculum learning, Human-in-the-loop machine learning}
}

@article{DBLP:journals/corr/abs-2210-03050,
  publtype={informal},
  author={Dieuwke Hupkes and Mario Giulianelli and Verna Dankers and Mikel Artetxe and Yanai Elazar and Tiago Pimentel and Christos Christodoulopoulos and Karim Lasri and Naomi Saphra and Arabella Sinclair and Dennis Ulmer and Florian Schottmann and Khuyagbaatar Batsuren and Kaiser Sun and Koustuv Sinha and Leila Khalatbari and Maria Ryskina and Rita Frieske and Ryan Cotterell and Zhijing Jin},
  title={State-of-the-art generalisation research in NLP: a taxonomy and review},
  year={2022},
  cdate={1640995200000},
  journal={CoRR},
  volume={abs/2210.03050},
  url={https://doi.org/10.48550/arXiv.2210.03050},
}





% RELATED WORK ---------------------------------------------------------------
@article{yang2021oodsurvey,
  title={Generalized Out-of-Distribution Detection: A Survey},
  author={Yang, Jingkang and Zhou, Kaiyang and Li, Yixuan and Liu, Ziwei},
  journal={arXiv preprint arXiv:2110.11334},
  year={2021}
}

@article{Chen2023FineTuningDG,
  title={Fine-Tuning Deteriorates General Textual Out-of-Distribution Detection by Distorting Task-Agnostic Features},
  author={Sishuo Chen and Wenkai Yang and Xiaohan Bi and Xu Sun},
  journal={EACL},
  year={2023}
}

@misc{rawat2021pnpood,
      title={PnPOOD : Out-Of-Distribution Detection for Text Classification via Plug andPlay Data Augmentation}, 
      author={Mrinal Rawat and Ramya Hebbalaguppe and Lovekesh Vig},
      year={2021},
      eprint={2111.00506},
      archivePrefix={arXiv},
      primaryClass={cs.AI}
}

@article{DBLP:journals/corr/abs-2106-09022,
  publtype={informal},
  author={Jie Ren and Stanislav Fort and Jeremiah Liu and Abhijit Guha Roy and Shreyas Padhy and Balaji Lakshminarayanan},
  title={A Simple Fix to Mahalanobis Distance for Improving Near-OOD Detection},
  year={2021},
  cdate={1609459200000},
  journal={CoRR},
  volume={abs/2106.09022},
  url={https://arxiv.org/abs/2106.09022}
}

@inproceedings{duan-etal-2022-barle,
    title = "{BARLE}: Background-Aware Representation Learning for Background Shift Out-of-Distribution Detection",
    author = "Duan, Hanyu  and
      Yang, Yi  and
      Abbasi, Ahmed  and
      Tam, Kar Yan",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2022",
    month = dec,
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.findings-emnlp.53",
    pages = "750--764",
    abstract = "Machine learning models often suffer from a performance drop when they are applied to out-of-distribution (OOD) samples, i.e., those drawn far away from the training data distribution. Existing OOD detection work mostly focuses on identifying semantic-shift OOD samples, e.g., instances from unseen new classes. However, background-shift OOD detection, which identifies samples with domain or style-change, represents a more practical yet challenging task. In this paper, we propose Background-Aware Representation Learning (BARLE) for background-shift OOD detection in NLP. Specifically, we generate semantics-preserving background-shifted pseudo OOD samples from pretrained masked language models. We then contrast the in-distribution (ID) samples with their pseudo OOD counterparts. Unlike prior semantic-shift OOD detection work that often leverages an external text corpus, BARLE only uses ID data, which is more flexible and cost-efficient. In experiments across several text classification tasks, we demonstrate that BARLE is capable of improving background-shift OOD detection performance while maintaining ID classification accuracy. We further investigate the properties of the generated pseudo OOD samples, uncovering the working mechanism of BARLE.",
}

@misc{zhang2023unsupervised,
      title={Unsupervised Evaluation of Out-of-distribution Detection: A Data-centric Perspective}, 
      author={Yuhang Zhang and Weihong Deng and Liang Zheng},
      year={2023},
      eprint={2302.08287},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

@InProceedings{pmlr-v15-bengio11b,
  title = 	 {Deep Learners Benefit More from Out-of-Distribution Examples},
  author = 	 {Bengio, Yoshua and Bastien, Frédéric and Bergeron, Arnaud and Boulanger–Lewandowski, Nicolas and Breuel, Thomas and Chherawala, Youssouf and Cisse, Moustapha and Côté, Myriam and Erhan, Dumitru and Eustache, Jeremy and Glorot, Xavier and Muller, Xavier and Pannetier Lebeuf, Sylvain and Pascanu, Razvan and Rifai, Salah and Savard, François and Sicard, Guillaume},
  booktitle = 	 {Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
  pages = 	 {164--172},
  year = 	 {2011},
  editor = 	 {Gordon, Geoffrey and Dunson, David and Dudík, Miroslav},
  volume = 	 {15},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Fort Lauderdale, FL, USA},
  month = 	 {11--13 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v15/bengio11b/bengio11b.pdf},
  url = 	 {https://proceedings.mlr.press/v15/bengio11b.html},
  abstract = 	 {Recent theoretical and empirical work in statistical machine learning has demonstrated the potential of learning algorithms for deep architectures, i.e., function classes obtained by composing multiple levels of representation. The hypothesis evaluated here is that intermediate levels of representation, because they can be shared across tasks and examples from different but related distributions, can yield even more benefits. Comparative experiments were performed on a large-scale handwritten character recognition setting with 62 classes (upper case, lower case, digits), using both a multi-task setting and perturbed examples in order to obtain out-of-distribution examples. The results agree with the hypothesis, and show that a deep learner did <em>beat previously published results and reached human-level performance</em>.}
}

@inproceedings{zhouetal2021contrastive,
    title = "Contrastive Out-of-Distribution Detection for Pretrained Transformers",
    author = "Zhou, Wenxuan  and
      Liu, Fangyu  and
      Chen, Muhao",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2021",
    address = "Online and Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.emnlp-main.84",
    doi = "10.18653/v1/2021.emnlp-main.84",
    pages = "1100--1111",
    abstract = "Pretrained Transformers achieve remarkable performance when training and test data are from the same distribution. However, in real-world scenarios, the model often faces out-of-distribution (OOD) instances that can cause severe semantic shift problems at inference time. Therefore, in practice, a reliable model should identify such instances, and then either reject them during inference or pass them over to models that handle another distribution. In this paper, we develop an unsupervised OOD detection method, in which only the in-distribution (ID) data are used in training. We propose to fine-tune the Transformers with a contrastive loss, which improves the compactness of representations, such that OOD instances can be better differentiated from ID ones. These OOD instances can then be accurately detected using the Mahalanobis distance in the model{'}s penultimate layer. We experiment with comprehensive settings and achieve near-perfect OOD detection performance, outperforming baselines drastically. We further investigate the rationales behind the improvement, finding that more compact representations through margin-based contrastive learning bring the improvement. We release our code to the community for future research.",
}

@inproceedings{Podolskiy2021RevisitingMD,
  title={Revisiting Mahalanobis Distance for Transformer-Based Out-of-Domain Detection},
  author={A. V. Podolskiy and Dmitry Lipin and A. Bout and E. Artemova and Irina Piontkovskaya},
  booktitle={AAAI Conference on Artificial Intelligence},
  year={2021}
}

@inproceedings{hendrycks-etal-2020-pretrained,
    title = "Pretrained Transformers Improve Out-of-Distribution Robustness",
    author = "Hendrycks, Dan  and
      Liu, Xiaoyuan  and
      Wallace, Eric  and
      Dziedzic, Adam  and
      Krishnan, Rishabh  and
      Song, Dawn",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.244",
    doi = "10.18653/v1/2020.acl-main.244",
    pages = "2744--2751",
    abstract = "Although pretrained Transformers such as BERT achieve high accuracy on in-distribution examples, do they generalize to new distributions? We systematically measure out-of-distribution (OOD) generalization for seven NLP datasets by constructing a new robustness benchmark with realistic distribution shifts. We measure the generalization of previous models including bag-of-words models, ConvNets, and LSTMs, and we show that pretrained Transformers{'} performance declines are substantially smaller. Pretrained transformers are also more effective at detecting anomalous or OOD examples, while many previous models are frequently worse than chance. We examine which factors affect robustness, finding that larger models are not necessarily more robust, distillation can be harmful, and more diverse pretraining data can enhance robustness. Finally, we show where future work can improve OOD robustness.",
}

@article{yang2022openood,
      title={OpenOOD: Benchmarking Generalized Out-of-Distribution Detection}, 
      author={Jingkang Yang and Pengyun Wang and Dejian Zou and Zitang Zhou and Kunyuan Ding and Wenxuan Peng and Haoqi Wang and Guangyao Chen and Bo Li and Yiyou Sun and Xuefeng Du and Kaiyang Zhou and Wayne Zhang and Dan Hendrycks and Yixuan Li and Ziwei Liu},
      year={2022},
      eprint={2210.07242},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
  journal={NeurIPS}
}

@misc{arora2021types,
      title={Types of Out-of-Distribution Texts and How to Detect Them}, 
      author={Udit Arora and William Huang and He He},
      year={2021},
      journal={EMNLP},
      eprint={2109.06827},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@inproceedings{ZongSMCLCC18,
  author={Bo Zong and Qi Song and Martin Renqiang Min and Wei Cheng and Cristian Lumezanu and Dae-ki Cho and Haifeng Chen},
  title={Deep Autoencoding Gaussian Mixture Model for Unsupervised Anomaly Detection},
  year={2018},
  cdate={1514764800000},
  url={https://openreview.net/forum?id=BJJLHbb0-},
  booktitle={ICLR (Poster)}
}

@inproceedings{
liang2018enhancing,
title={Enhancing The Reliability of Out-of-distribution Image Detection in Neural Networks},
author={Shiyu Liang and Yixuan Li and R. Srikant},
booktitle={International Conference on Learning Representations},
year={2018},
url={https://openreview.net/forum?id=H1VGkIxRZ},
}

@misc{xu2022conal,
      title={CoNAL: Anticipating Outliers with Large Language Models}, 
      author={Albert Xu and Xiang Ren and Robin Jia},
      year={2022},
      eprint={2211.15718},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{yang2022gluex,
  publtype={informal},
  author={Linyi Yang and Shuibai Zhang and Libo Qin and Yafu Li and Yidong Wang and Hanmeng Liu and Jindong Wang and Xing Xie and Yue Zhang},
  title={GLUE-X: Evaluating Natural Language Understanding Models from an Out-of-distribution Generalization Perspective},
  year={2022},
  cdate={1640995200000},
  journal={CoRR},
  volume={abs/2211.08073},
  url={https://doi.org/10.48550/arXiv.2211.08073},
}

@misc{kuan2022basics,
      title={Back to the Basics: Revisiting Out-of-Distribution Detection Baselines}, 
      author={Johnson Kuan and Jonas Mueller},
      year={2022},
      eprint={2207.03061},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{arora\-etal\-2021\-types,
    title = "Types of Out-of-Distribution Texts and How to Detect Them",
    author = "Arora, Udit  and
      Huang, William  and
      He, He",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2021",
    address = "Online and Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.emnlp-main.835",
    doi = "10.18653/v1/2021.emnlp-main.835",
    pages = "10687--10701",
    abstract = "Despite agreement on the importance of detecting out-of-distribution (OOD) examples, there is little consensus on the formal definition of the distribution shifts of OOD examples and how to best detect them. We categorize these examples as exhibiting a background shift or semantic shift, and find that the two major approaches to OOD detection, calibration and density estimation (language modeling for text), have distinct behavior on these types of OOD data. Across 14 pairs of in-distribution and OOD English natural language understanding datasets, we find that density estimation methods consistently beat calibration methods in background shift settings and perform worse in semantic shift settings. In addition, we find that both methods generally fail to detect examples from challenge data, indicating that these examples constitute a different type of OOD data. Overall, while the categorization we apply explains many of the differences between the two methods, our results call for a more explicit definition of OOD to create better benchmarks and build detectors that can target the type of OOD data expected at test time.",
}









% DATASETS --------------------------------------------------
@article{misra2022news,
  title={News Category Dataset},
  author={Misra, Rishabh},
  journal={arXiv preprint arXiv:2209.11429},
  year={2022}
}

@misc{Newsgroups20,
  added-at = {2012-05-12T16:39:37.000+0200},
  author = {{McGraw Hill}},
  biburl = {https://www.bibsonomy.org/bibtex/2133313f1254d09b6ae89da442b62f3bc/lopusz_kdd},
  interhash = {77445767e91f49215dc81ff8236ca9d5},
  intrahash = {133313f1254d09b6ae89da442b62f3bc},
  keywords = {dataset},
  timestamp = {2012-05-12T18:24:34.000+0200},
  title = { 20 Newsgroups Dataset},
  url = {http://people.csail.mit.edu/jrennie/20Newsgroups/},
  year = {1995}
}


@inproceedings{dimosthenis-etal-2022-twitter,
    title = "{T}witter {T}opic {C}lassification",
    author = "Antypas, Dimosthenis  and
    Ushio, Asahi  and
    Camacho-Collados, Jose  and
    Neves, Leonardo  and
    Silva, Vitor  and
    Barbieri, Francesco",
    booktitle = "Proceedings of the 29th International Conference on Computational Linguistics",
    month = oct,
    year = "2022",
    address = "Gyeongju, Republic of Korea",
    publisher = "International Committee on Computational Linguistics"
}

@inproceedings{socher,
    title = "Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank",
    author = "Socher, Richard  and
      Perelygin, Alex  and
      Wu, Jean  and
      Chuang, Jason  and
      Manning, Christopher D.  and
      Ng, Andrew  and
      Potts, Christopher",
    booktitle = "Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing",
    month = oct,
    year = "2013",
    address = "Seattle, Washington, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/D13-1170",
    pages = "1631--1642"
}

@article{zhangCharacterlevelConvolutionalNetworks2015,
  archivePrefix = {arXiv},
  eprinttype = {arxiv},
  eprint = {1509.01626},
  primaryClass = {cs},
  title = {Character-Level {{Convolutional Networks}} for {{Text Classification}}},
  abstract = {This article offers an empirical exploration on the use of character-level convolutional networks (ConvNets) for text classification. We constructed several large-scale datasets to show that character-level convolutional networks could achieve state-of-the-art or competitive results. Comparisons are offered against traditional models such as bag of words, n-grams and their TFIDF variants, and deep learning models such as word-based ConvNets and recurrent neural networks.},
  journal = {arXiv:1509.01626 [cs]},
  author = {Zhang, Xiang and Zhao, Junbo and LeCun, Yann},
  month = sep,
  year = {2015},
}

@InProceedings{maas-EtAl:2011:ACL-HLT2011,
  author    = {Maas, Andrew L.  and  Daly, Raymond E.  and  Pham, Peter T.  and  Huang, Dan  and  Ng, Andrew Y.  and  Potts, Christopher},
  title     = {Learning Word Vectors for Sentiment Analysis},
  booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},
  month     = {June},
  year      = {2011},
  address   = {Portland, Oregon, USA},
  publisher = {Association for Computational Linguistics},
  pages     = {142--150},
  url       = {http://www.aclweb.org/anthology/P11-1015}
}

@misc{LanguageDataset,
  title = {A Dataset for Language Detection},
  howpublished = {\url{https://www.kaggle.com/datasets/basilb2s/language-detection}},
  note = {Accessed: 2023-04-15},
  year = {2021},
  author = {Basil Saji}
}

% METHODS ---------------------------------------------------------------
@inproceedings{liu2021energybased,
  author={Weitang Liu and Xiaoyun Wang and John D. Owens and Yixuan Li},
  title={Energy-based Out-of-distribution Detection},
  year={2020},
  cdate={1577836800000},
  url={https://proceedings.neurips.cc/paper/2020/hash/f5496252609c43eb8a3d147ab9b9c006-Abstract.html},
  booktitle={NeurIPS}
}

@inproceedings{wang2022vim,
  publtype={informal},
  author={Haoqi Wang and Zhizhong Li and Litong Feng and Wayne Zhang},
  title={ViM: Out-Of-Distribution with Virtual-logit Matching},
  year={2022},
  cdate={1640995200000},
  journal={CoRR},
  volume={abs/2203.10807},
  url={https://doi.org/10.48550/arXiv.2203.10807},
  booktitle={NeurIPS}
}

@inproceedings{sun2022outofdistribution,
  title={Out-of-distribution detection with deep nearest neighbors},
  author={Sun, Yiyou and Ming, Yifei and Zhu, Xiaojin and Li, Yixuan},
  booktitle={International Conference on Machine Learning},
  pages={20827--20840},
  year={2022},
  organization={PMLR}
}

@misc{hendrycks2022scaling,
      title={Scaling Out-of-Distribution Detection for Real-World Settings}, 
      author={Dan Hendrycks and Steven Basart and Mantas Mazeika and Andy Zou and Joe Kwon and Mohammadreza Mostajabi and Jacob Steinhardt and Dawn Song},
      year={2022},
      eprint={1911.11132},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

@inproceedings{huang2021importance,
  publtype={informal},
  author={Rui Huang and Andrew Geng and Yixuan Li},
  title={On the Importance of Gradients for Detecting Distributional Shifts in the Wild},
  year={2021},
  cdate={1609459200000},
  journal={CoRR},
  volume={abs/2110.00218},
  url={https://arxiv.org/abs/2110.00218},
  booktitle={NeurIPS}
}

@inproceedings{sun2021react,
  author={Yiyou Sun and Chuan Guo and Yixuan Li},
  title={ReAct: Out-of-distribution Detection With Rectified Activations},
  year={2021},
  cdate={1609459200000},
  pages={144-157},
  url={https://proceedings.neurips.cc/paper/2021/hash/01894d6f048493d2cacde3c579c315a3-Abstract.html},
  booktitle={NeurIPS}
}

@misc{sun2022dice,
      title={DICE: Leveraging Sparsification for Out-of-Distribution Detection}, 
      author={Yiyou Sun and Yixuan Li},
      year={2022},
      eprint={2111.09805},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{hendrycks17baseline,
  author    = {Dan Hendrycks and Kevin Gimpel},
  title     = {A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks},
  journal = {Proceedings of International Conference on Learning Representations},
  year = {2017},
}

% MODELS -----------------------------------------------------------------
@article{DBLP:journals/corr/abs-1907-11692,
  author    = {Yinhan Liu and
               Myle Ott and
               Naman Goyal and
               Jingfei Du and
               Mandar Joshi and
               Danqi Chen and
               Omer Levy and
               Mike Lewis and
               Luke Zettlemoyer and
               Veselin Stoyanov},
  title     = {RoBERTa: {A} Robustly Optimized {BERT} Pretraining Approach},
  journal   = {CoRR},
  volume    = {abs/1907.11692},
  year      = {2019},
  url       = {http://arxiv.org/abs/1907.11692},
  archivePrefix = {arXiv},
  eprint    = {1907.11692},
  timestamp = {Thu, 01 Aug 2019 08:59:33 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1907-11692.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


% MISC -----------------------------------------------------------------
@INPROCEEDINGS{6120320,
  author={Raskutti, Garvesh and Wainwright, Martin J. and Yu, Bin},
  booktitle={2011 49th Annual Allerton Conference on Communication, Control, and Computing (Allerton)}, 
  title={Early stopping for non-parametric regression: An optimal data-dependent stopping rule}, 
  year={2011},
  volume={},
  number={},
  pages={1318-1325},
  doi={10.1109/Allerton.2011.6120320}}

@INPROCEEDINGS{8624183,
  author={Zhang, Zijun},
  booktitle={2018 IEEE/ACM 26th International Symposium on Quality of Service (IWQoS)}, 
  title={Improved Adam Optimizer for Deep Neural Networks}, 
  year={2018},
  volume={},
  number={},
  pages={1-2},
  doi={10.1109/IWQoS.2018.8624183}}

@inproceedings{NIPS2017_3f5ee243,
 author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf},
 volume = {30},
 year = {2017}
}

@article{DBLP:journals/corr/abs-2303-18223,
  publtype={informal},
  author={Wayne Xin Zhao and Kun Zhou and Junyi Li and Tianyi Tang and Xiaolei Wang and Yupeng Hou and Yingqian Min and Beichen Zhang and Junjie Zhang and Zican Dong and Yifan Du and Chen Yang and Yushuo Chen and Zhipeng Chen and Jinhao Jiang and Ruiyang Ren and Yifan Li and Xinyu Tang and Zikang Liu and Peiyu Liu and Jian-Yun Nie and Ji-Rong Wen},
  title={A Survey of Large Language Models},
  year={2023},
  cdate={1672531200000},
  journal={CoRR},
  volume={abs/2303.18223},
  url={https://doi.org/10.48550/arXiv.2303.18223},
}

@article{10.1145/3522593,
author = {Wang, Yu and Tong, Hanghang and Zhu, Ziye and Li, Yun},
title = {Nested Named Entity Recognition: A Survey},
year = {2022},
issue_date = {December 2022},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {16},
number = {6},
issn = {1556-4681},
url = {https://doi.org/10.1145/3522593},
doi = {10.1145/3522593},
abstract = {With the rapid development of text mining, many studies observe that text generally contains a variety of implicit information, and it is important to develop techniques for extracting such information. Named Entity Recognition (NER), the first step of information extraction, mainly identifies names of persons, locations, and organizations in text. Although existing neural-based NER approaches achieve great success in many language domains, most of them normally ignore the nested nature of named entities. Recently, diverse studies focus on the nested NER problem and yield state-of-the-art performance. This survey attempts to provide a comprehensive review on existing approaches for nested NER from the perspectives of the model architecture and the model property, which may help readers have a better understanding of the current research status and ideas. In this survey, we first introduce the background of nested NER, especially the differences between nested NER and traditional (i.e., flat) NER. We then review the existing nested NER approaches from 2002 to 2020 and mainly classify them into five categories according to the model architecture, including early rule-based, layered-based, region-based, hypergraph-based, and transition-based approaches. We also explore in greater depth the impact of key properties unique to nested NER approaches from the model property perspective, namely entity dependency, stage framework, error propagation, and tag scheme. Finally, we summarize the open challenges and point out a few possible future directions in this area. This survey would be useful for three kinds of readers: (i) Newcomers in the field who want to learn about NER, especially for nested NER. (ii) Researchers who want to clarify the relationship and advantages between flat NER and nested NER. (iii) Practitioners who just need to determine which NER technique (i.e., nested or not) works best in their applications.},
journal = {ACM Trans. Knowl. Discov. Data},
month = {jul},
articleno = {108},
numpages = {29},
keywords = {information extraction, natural language processing, text mining, Nested named entity recognition, named entity recognition}
}

@article{pandya2021question,
  title={Question answering survey: Directions, challenges, datasets, evaluation matrices},
  author={Pandya, Hariom A and Bhatt, Brijesh S},
  journal={arXiv preprint arXiv:2112.03572},
  year={2021}
}

@article{bag-of-words,
   author = {Yin Zhang and Rong Jin and Zhi-Hua Zhou},
   keywords = {Bag of words model ⋅,Object recognition ⋅,Rademacher complexity},
   title = {Understanding Bag-of-Words Model: A Statistical Framework},
   year = {2010},
   journal={International Journal of Machine Learning and Cybernetics},
}




























