% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{acl}
% \usepackage[review]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{multirow}
\usepackage{url}
\usepackage{hyperref}
\usepackage{graphicx}
%\DeclareEmphSequence{\gtfamily\sffamily\bfseries}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
\usepackage{CJKutf8}
\newcommand*{\ja}[1]{\begin{CJK}{UTF8}{ipxm}#1\end{CJK}}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

\newcommand{\yamada}[1]{\textcolor{magenta}{[Yamada: #1]}}
\newcommand{\bhline}{\noalign{\hrule height 1.2pt}}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Building a Buzzer-Quiz Answering System}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{
  Naoya Sugiura \hspace{2.0em}
  Kosuke Yamada \hspace{2.0em}
  Ryohei Sasano \\
  \textbf{Koichi Takeda} \hspace{2.0em}
  \textbf{Katsuhiko Toyama} \\
  Graduate School of Informatics, Nagoya University, Japan\\
  \texttt{\{sugiura.naoya.e7,yamada.kosuke.v1\}@s.mail.nagoya-u.ac.jp} \\
  \texttt{\{sasano,takedasu,toyama\}@i.nagoya-u.ac.jp} \\
}
  
\begin{document}
\maketitle
\begin{abstract}
A buzzer quiz is a genre of quiz in which multiple players simultaneously listen to a quiz being read aloud and respond it by buzzing in as soon as they can predict the answer.
Because incorrect answers often result in penalties, a buzzer-quiz answering system must not only predict the answer from only part of a question but also estimate the predicted answer's accuracy.
In this paper, we introduce two types of buzzer-quiz answering systems: (1) a system that directly generates an answer from part of a question by using an autoregressive language model; and (2) a system that first reconstructs the entire question by using an autoregressive language model and then determines the answer according to the reconstructed question.
We then propose a method to estimate the accuracy of the answers for each system by using the internal scores of each model.
\end{abstract}

\section{Introduction}
We use the term ``buzzer quiz'' to refer to a genre of quiz in which questioner reads quiz questions aloud and players answer by buzzing in as soon as they can predict the answer.
A well-known example of a similar format to what we call a buzzer quiz here is the U.S. TV program \emph{Jeopady!}, in which contestants must buzz in with a lock-out device before trying to answer a question.
However, in \emph{Jeopady!}, answers are only allowed after all the questions have been read aloud, whereas we assume a format in which answers are allowed while the questions are being read out.
Because of the importance of buzzing in quickly, players normally answer incomplete questions in buzzer quiz. 

\begin{table}[t!]
\centering
\small
\begin{tabular}{@{\ }l@{\ }} \bhline
\rule{0pt}{2.5ex}
\textbf{Q (75\% completeness):} Pete Rose and this player are \\
tied with ten 200-hit seasons each. 
This Japanese \\
outfielder played most of his career with the Mar\textcolor[gray]{0.7}{iners,} \\
\textcolor[gray]{0.7}{and currently plays for the Marlins.} \\
\rule{0pt}{2.5ex}
\textbf{Confidence score:} 0.991 \textbf{A:} Ichiro Suzuki \ \ \emph{\color{red} correct} \\ \bhline
\rule{0pt}{2.5ex}
\textbf{Q (25\% completeness)}: Pete Rose and this player are \\
tied with ten 2\textcolor[gray]{0.7}{00-hit seasons each. This Japanese} \\
\textcolor[gray]{0.7}{outfielder played most
of his career with the Mariners,} \\
\textcolor[gray]{0.7}{and currently plays for the Marlins.} \\
\rule{0pt}{2.5ex}
\textbf{Confidence score:} 0.125 \textbf{A:} Ty Cobb \ \ \emph{\color{blue} incorrect} \\ \bhline
\rule{0pt}{2.5ex}
\end{tabular}\vspace{-4.0ex}
\caption{Output example. Gray text indicates the unread portion of the question text. ``Completeness'' denotes the percentage of the question text that has been read, and the ``confidence score'' refers to a value indicating the likelihood of the predicted answer being correct.}\vspace{-2.5ex}
\label{tab:example}
\end{table}

Quizzes have been studied as open-domain question answering (QA) tasks because they do not limit the scope of knowledge.
However, the major datasets for open-domain QA tasks, like Natural Questions \cite{kwiatkowski-etal-2019-natural} and TriviaQA \cite{joshi-etal-2017-triviaqa} cointain complete questions. 
Consequently, systems built using those datasets \cite{karpukhin-etal-2020-dense,yamada-etal-2021-efficient,izacard-grave-2021-leveraging} are not designed to answer incomplete questions.
Furthermore, it is certainly crucial in buzzer quizzes to give correct answers, but it is also essential to consider the plausibility of a predicted answer based on the given question at that moment and to decide whether to actually respond.
For example, consider the question listed in Table \ref{tab:example} if it has not been read past the phrase ``200-hit.''
At that point, because other baseball players also hold records comparable to that of Pete Rose, it is difficult to narrow the answer down to a single candidate.
This makes the predicted answer at that moment more likely to be incorrect, so it would be better not to answer at that point.
On the other hand, once the question has been read further, the predicted answer converges to the correct answer, ``Ichiro Suzuki.''
Hence, to construct a more effective buzzer-quiz answering system, we need an indicator of a predicted answer's liklihood of being correct, which call a ``confidence score.''

We believe that the capability to respond to buzzer quizzes by answering incomplete questions could help replicate the human capacity to smoothly generate responses in a conversation by sequentially predicting the content of the dialogue.
In this study, we first constructed a buzzer-quiz answering system that produces appropriate answers for incomplete questions, and we propose the methods of calculating the confidence scores for two different models.
Specifically, we constructed two systems: the GPT (Inference) system, which directly generates answers in response to a question by using GPT \cite{radford-2018-improving}; and the GPT + DPR system, which generates answers through a retriever-reader approach using Dense Passage Retrieval (DPR) \cite{karpukhin-etal-2020-dense}, after completing the question via GPT. 
For the former system, we calculate a confidence score by using token output probabilities during answer generation, while for the latter system, we use scores that are used in the output of the model. 

\section{Proposed Method}
\label{sec:DPRandGPT}
We propose two types of buzzer-quiz answering systems based on open-domain QA systems. 
We also propose methods to estimate the accuracy of the answers in each system by using the internal scores in each model.

\subsection{Open-Domain QA System}
\label{sec:OpenDomain}
In open-domain QA, there are two mainstream approaches.
The first is a generation-based approach that generates answers directly in response to input questions.
A representative models is GPT \cite{radford-2018-improving}, which is a pretrained language model that is based on the Transformer decoder \cite{vaswani-2017-attention} and is trained to predict word sequences from a context by using a large text corpus.
Because of this property, GPT can be used in language generation tasks that involve generating text in response to input text.
In the case of QA, GPT can generate answers by formatting the input in such a way as to infer only the answer to a question.
Furthermore, because GPT often achieves higher performance through finetuning with datasets from downstream tasks, such finetuning can be applied to build QA models.

The second major open-domain QA approach is a retriever-reader approach that searches for documents related to a question and extracts the answer from the documents.
A representative model is the retriever-reader model, which uses DPR as the retriever.
DPR uses a dual encoder network with different BERT models \cite{devlin-2019-bert} for questions and documents.
When sentences are input to BERT, a special token \verb|[CLS]| is inserted at the beginning of a document, and the embedding representations for the question text and each document are obtained.
Then, documents are selected according to the semantic similarity calculated as the inner product of the obtained representations \cite{karpukhin-etal-2020-dense}.
In the reader, BERT predicts the relevant documents containing the correct answer and extracts the answer portion within a document. 
Specifically, it predicts the document that is most likely to contain the answer at the position of the token \verb|[CLS]|. 
Then, it performs the answer-portion extraction from for the predicted document and determines the start and end points of the token sequence that forms the answer.

\subsection{Buzzer-Quiz Answering Systems}
The effectiveness of the open-domain QA systems that answer complete questions has been confirmed, but their effectiveness for a buzzer-quiz answering system remains unclear, because such a system requires to answer incomplete questions.
Generally, when only part of a question is given, the nature of the problem differs significantly from the case of a complete question, because there may be multiple possible answers, or the necessary information to determine the answer might not be available yet.

In this study, we constructed two buzzer-quiz answering systems: one that relies solely on inference via GPT, called the \textbf{GPT (Inference)} system, and another that uses GPT for question completion and applies the retriever-reader approach with DPR, called the \textbf{GPT + DPR} system.
For the GPT (Inference) system, the designed input format is ``[question text] + `/the answer is','' which prompts the model to generate the answer within the single quotation marks, which is then used as the predicted answer.
The propose of inserting a slash `/' between the question text and ``the answer is'' is to make the model to recognize the boundary of the question text, which prevents  completion of incomplete questions.
For the GPT + DPR system, an incomplete question is input to the GPT to complete the question text, and the resulting complete question is then used as input for the DPR-based retriever-reader model to generate the answer. 

\subsection{Confidence Scores}
\label{sec:ConfidenceScore}
Next, we propose to calculate the confidence scores for predicted answers by using the internal scores that each model uses when it generates their outputs for the buzzer-quiz answering system.
Here, the confidence score means an indicator for judging whether a predicted answer is correct.
For higher values of our proposed confidence scores, we expect a higher percentage of correct answers.

For the GPT (Inference) model, we use the generation probability of the first token in the predicted answer (referred to as the \textbf{generation score}) as the confidence score.
When given a sentence's first $n$ tokens during sentence completion, GPT outputs the $(n+1)$-th token from the vocabulary with the highest generation score.
The first token largely determines a direction of the answer in buzzer quiz, because the answer is often comprises a small number of tokens. 
Hence, we adopt only first token's generation score as the confidence score.

As for the GPT + DPR model, three internal scores can be used as confidence scores: the \textbf{document score} and the \textbf{extraction score} calculated by the reader, as well as their arithmetic mean, the \textbf{average score}.
In the reader, each \verb|[CLS]| token in a document is scored through a learned linear layer, and the document with the highest score is selected; this is the document score. 
Then, the model extracts the span containing the answer from the selected document by calculating a span score, which comprises a start score and an end score.
The extraction score is the sum of these start and end scores.

\section{Experiments}
We conducted two experiments: an evaluation of the proposed buzzer-quiz answering system's accuracy, and an investigation of effectiveness of the confidence scores for each model.
We define the question completeness as $x$\% when a question is truncated after the first $x$\% of the text in terms of the character count.
For the accuracy verification, we applied the GPT (Inference) and the GPT + DPR models to questions with completeness levels of 25\%, 50\%, 75\%, and 100\%.
For investigation of the confidence scores' effectiveness, we evaluated the confidence scores for each model by examining the relationship between the confidence scores and the accuracy at each level of question completeness.


\subsection{Settings}
\begin{table}[t!]
\centering
\small
\begin{tabular}{cc|rr} \bhline
Subset & Source & \multicolumn{1}{|c}{Size} & \multicolumn{1}{c}{Length} \\ \bhline
\multirow{2}{*}{Train} & AIO & 17,735 &  48.2\\
& Minhaya & 35,149 & 64.8 \\ \hline
Dev & AIO & 1,000 & 46.9 \\ \hline
Test & AIO & 2,000 & 51.6 \\ \bhline
\end{tabular}\vspace{-2ex}
\caption{Overview of the datasets. ``Length'' means the average number of characters for the questions.}\vspace{-2ex}
\label{tab:data}
\end{table}

\paragraph{Datasets}
 We used the 2nd AIO Official Dataset (AIO),\footnote{\url{https://sites.google.com/view/project-aio/dataset}} which contains past questions from Japanese quiz competitions, and we crawled past questions from the Japanese quiz application ``Minna de Hayaoshi Quiz'' (Minhaya).\footnote{\url{https://livequiz.work/minhaya1/}}
For each dataset, Table \ref{tab:data} lists the number of quiz-answer pairs and the average number of characters for the questions.
Note that, in addition to quiz-answer pairs, the DPR training required positive and negative documents.
Accordingly, DPR was trained using only the AIO dataset, whereas the Minhaya dataset was used only for training GPT.

\paragraph{Comparison Models}
We compared both models, GPT (Inference) and GPT + DPR, in the accuracy verification.
In the investigation of confidence score effectiveness, for GPT (Inference), we used the generation score; in contrast, for GPT + DPR, we all three scores, i.e., the document score, extraction score, and average score.

We used the Japanese GPT model on Hugging Face \cite{wolf-2020-transformers}\footnote{\url{https://huggingface.co/rinna/japanese-gpt-1b}} and a DPR model based on Japanese BERT-large,\footnote{\url{https://huggingface.co/cl-tohoku/bert-large-japanese}} which learned the Japanese Wikipedia corpus.\footnote{\url{https://github.com/cl-tohoku/AIO2_DPR_baseline}}
For GPT (Inference), we finetuned the model on the training set with the input format ``[question text] + `/ the answer is' [answer].'' 
For GPT + DPR, GPT was finetuned using only the questions from the training set. 
In both cases, the training was conducted for 5 epochs.
DPR was based on Japanese BERT\footnote{\url{https://huggingface.co/cl-tohoku/bert-large-japanese}} for both the retriever and reader components. 
The retriever was trained for 5 epochs with a batch size of 128 and a learning rate of 1e-5, while the reader was trained for 3 epochs with a batch size of 8 and a learning rate of 2e-5.

\paragraph{Metrics}
In the accuracy verification, the correctness of the predicted answer was assessed in terms of exact matching.
In the investigation of confidence score effectiveness, we created curves of the correct answer rate with respect to the answer generation rate, and we evaluated the effectiveness in terms of the area under the curve (AUC). 
Here, the answer generation rate was the proportion of times that the system actually provided an answer.
If the models only answered questions for which the confidence score exceeded a threshold $\alpha$, we can control the answer rate by changing $\alpha$. 
On the other hand, the correct answer rate was the proportion of correct answers among the answers output by the models.
If $\alpha$ is set to a value below 0, the answer rate will coincide with the overall correct answer rate of the system. 
As $\alpha$ increases, only questions with high confidence scores will be answered, so the correct answer rate will be expected to increase.

\subsection{Accuracy Verification}
\label{sec:experimentA}

\begin{table}[t!]
\centering
\small
\begin{tabular}{llcccc} \bhline
Model & 25\% & 50\% & 75\% & 100\% \\ \bhline
GPT (Inference) & 11.9 & 27.9 & 45.6 & 56.2 \\
GPT + DPR & 11.9 & 28.8 & 45.9 & 62.0 \\ \bhline
\end{tabular}\vspace{-2ex}
\caption{Results of accuracy verification. The x\% represents the question completeness.}\vspace{-0.5ex}
\label{tab:EM}
\end{table}

Table \ref{tab:EM} lists the accuracies for the GPT (Inference) and GPT + DPR models for each level of question completeness.
As the question completeness decreased, the correct answer rate also decreased, but the rate of decrease was not proportional.  
From 100\% to 75\%, the decline was relatively gentle.
This was likely because many important words that determine the answer appear in the first half of a question, whereas cases with information-rich words appearing in the latter half of a question are relatively rare.
Comparing the scores of the two models, we see that GPT + DPR performed better when the question completeness was 100\%. 
When the questions were incomplete, however, there was no significant difference in performance between the two models was observed.

\subsection{Investigation of Confidence Score Effectiveness}
\label{sec:experimentB}

\begin{table}[t!]
\centering
\small
\vspace{0.5ex}
\begin{tabular}{@{\ }l@{\ \ }l@{\ \ }c@{\ \ }c@{\ \ }c@{\ \ }c@{\ }} \bhline
Model & Score & 25\% & 50\% & 75\% & 100\% \\ \bhline
GPT (Inference) & generation score & \textbf{41.4} & \textbf{63.6} & \textbf{81.2} & \textbf{85.9} \\ \hline
& document score & 31.8 & 58.1 & 77.3 & 84.8 \\
GPT + DPR & extraction score & 25.0 & 51.3 & 70.0 & 84.1 \\
& average score & 29.1 & 56.1 & 75.8 & 84.0 \\
\bhline
\end{tabular}\vspace{-2ex}
\caption{AUC values for each level of question completeness. ``Score'' means the internal scores we used.}\vspace{-2.5ex}
\label{tab:AUC}
\end{table}

\begin{figure}[t!]
\centering
\includegraphics[width=0.9\linewidth]{PRC_Eng_2.pdf}\vspace{-2ex}
\caption{Curves of the correct answer rate vs. the answer generation rate. The x\% represents the question completeness.}\vspace{-2ex}
\label{fig:AUC}
\end{figure}

Table \ref{tab:AUC} lists the AUC values for each level of question completeness.
Among the three confidence scores for GPT + DPR, using the document score yielded in the highest AUC.
Furthermore, among all the results, the generation score for GPT (Inference) achieved the highest AUC.
Next, because the document score had the highest AUC for GPT + DPR, we used it to compare the correct answer rate vs. answer generation rate curves of the GPT (Inference) model and the GPT + DPR models.
Figure \ref{fig:AUC} shows the results.

For all settings, we can observe that the accuracy was increased by limiting the questions to be answered to only those with high confidence scores, thus confirming the effectiveness of the confidence scores.
Comparing GPT (Inference) and GPT + DPR, as listed in Table \ref{tab:EM}, the accuracy at an answer rate of 1.0 was higher for GPT + DPR when the question completeness was 100\%, and equivalent in for less-complete questions. 
When the answer rate was less than 0.8, however, GPT (Inference) had a higher accuracy in all cases.
This difference was more obvious when both the question completeness and the answer rate were low. 
For example, in the case of 25\% question completeness and an answer rate of 0.1, the accuracy of GPT + DPR is around 0.5, whereas that of GPT (Inference) was around 0.8, thus showing a significant difference.
Accordingly, we can conclude that the GPT (Inference) model is more suitable for buzzer quizzes.

\begin{table*}[!t]
\small
\centering
\begin{tabular}{l|l}
\bhline
 \multicolumn{1}{c}{Japanese} & \multicolumn{1}{c}{English} \\\bhline

\begin{tabular}[l]{@{}l@{}}
 \ja{ごはんの上にハンバーグと}\textcolor[gray]{0.7}{\ja{目玉焼きを乗せ、グレービー}}\\
 \textcolor[gray]{0.7}{\ja{ソースをかけたハワイの名物料理は何でしょう?}}\\
 \textbf{Confidence score:} 0.996 \textbf{A:} \ja{ロコモコ} \ \ \emph{\color{red} correct}
\end{tabular}
 &
 \begin{tabular}[l]{@{}l@{}}
This rice dish is made by topping with a hamburger\\
and \textcolor[gray]{0.7}{a fried egg, and then drizzling it with gravy sauce.} \\\textcolor[gray]{0.7}{This dish originated in Hawaii.} \textcolor[gray]{0.7}{What is this?}\\
 \textbf{Confidence score:} 0.996 \textbf{A:} \ja{Loco moco} \ \ \emph{\color{red} correct}
 \end{tabular}\\\hline
 
 \begin{tabular}[l]{@{}l@{}} 
\ja{オーストリアの首都}\textcolor[gray]{0.7}{\ja{はウィーンですが、オーストラ}}\\
\textcolor[gray]{0.7}{\ja{リアの首都はどこでしょう?}}\\
\textbf{Confidence score:} 0.982 \textbf{A:} \ja{キャンベラ} \ \ \emph{\color{blue} incorrect}
 \end{tabular}
 &
 \begin{tabular}[l]{@{}l@{}} 
 The capital of Austria \textcolor[gray]{0.7}{is Vienna, but what is the} \\
 \textcolor[gray]{0.7}{capital of Australia?}\\
 \textbf{Confidence score:} 0.982 \textbf{A:} Canberra \ \ \emph{\color{blue} incorrect}
 \end{tabular}\\\bhline
\end{tabular}\vspace{-1.5ex}
\caption{Original Japanese outputs and its English translation at question completeness = 25\%.
``English'' is a English translation of original outputs that aims to preserve the information contained in the original Japanese questions.}
\label{tb:gene2}
\end{table*}

\begin{table*}[!t]
\small
\centering
\begin{tabular}{l|l}
\bhline
 \multicolumn{1}{c}{Japanese} & \multicolumn{1}{c}{English} \\\bhline

\begin{tabular}[l]{@{}l@{}}
 \ja{約5年の歳月をかけてシスティーナ礼拝堂の祭壇に描か}\\
 \ja{れた、ミケランジェロの代表}\textcolor[gray]{0.7}{\ja{作である絵画は何でしょう?}}\\
  \textbf{Confidence score:} 0.991 \textbf{A:} \ja{最後の審判} \ \ \emph{\color{red} correct}
\end{tabular}
 &
 \begin{tabular}[l]{@{}l@{}}
This painting was created over the span of about five\\
years in the Sistine Chapel. Now, this is known as\\
one of Michelangelo's master\textcolor[gray]{0.7}{pieces. What is this?}\\
  \textbf{Confidence score:} 0.991 \textbf{A:} The Last Judgment  \ \ \emph{\color{red} correct}
 \end{tabular}\\\hline
 
 \begin{tabular}[l]{@{}l@{}} 
\ja{1985年に発売され、全世界で 4000万本以上を売り上げた}\\
\ja{という任天堂ファミリーコンピュータのゲームで、}\\
\textcolor[gray]{0.7}{\ja{「スーマリ」などと略されるものは何?}}\\
\textbf{Confidence score:} 0.955 \textbf{A:} \ja{ドンキーコング} \ \ \emph{\color{blue} incorrect}
 \end{tabular}
 &
 \begin{tabular}[l]{@{}l@{}} 
 This game was launched for the Nintendo Family \\
 Computer in 1985 and has sold 40 million copies. \\
 \textcolor[gray]{0.7}{It is often also referred to by the abbreviation} \\
 \textcolor[gray]{0.7}{"Su-Mari". What is this?}\\
 \textbf{Confidence score:} 0.955 \textbf{A:} \ja{Donkey Kong} \ \ \emph{\color{blue} incorrect}
 \end{tabular}\\\bhline
\end{tabular}\vspace{-1.5ex}
\caption{Original Japanese outputs and its English translation at question completeness = 75\%.}
\label{tb:gene}
\end{table*}

\section{Conclusion}
In this study, we constructed two models for answering buzzer quiz questions, which have not been considered in previous research: GPT (Inference) and GPT + DPR. 
Then, we evaluated the accuracy for various levels of question completeness. 
Furthermore, we investigated the relationship between the model's internal scores, which were treated as confidence scores, and the accuracy; as a result, the validity of using the internal scores of the models as confidence scores was confirmed.

\section{Limitations}


\section{Future Work}
In the future, to increase the accuracy rate, we consider the use of more powerful models like ChatGPT or FiD \cite{izacard-grave-2021-leveraging}, and comparing it with human players to validate the differences between human performance and the model's one.

\section*{Acknowledgements}
This work was partly supported by JSPS KAKENHI Grant Number 21H04901.

% Entries for the entire Anthology, followed by custom entries
\bibliography{reference}
\bibliographystyle{acl_natbib}

\end{document}
