@inproceedings{watson-etal-2023-hiddentables,
title = "{H}idden{T}ables and {P}y{QT}ax: A Cooperative Game and Dataset For {T}able{QA} to Ensure Scale and Data Privacy Across a Myriad of Taxonomies",
author = "Watson, William and
Cho, Nicole and
Balch, Tucker and
Veloso, Manuela",
editor = "Bouamor, Houda and
Pino, Juan and
Bali, Kalika",
booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
month = dec,
year = "2023",
address = "Singapore",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.emnlp-main.442",
doi = "10.18653/v1/2023.emnlp-main.442",
pages = "7144--7159",
abstract = "A myriad of different Large Language Models (LLMs) face a common challenge in contextually analyzing table question-answering tasks. These challenges are engendered from (1) finite context windows for large tables, (2) multi-faceted discrepancies amongst tokenization patterns against cell boundaries, and (3) various limitations stemming from data confidentiality in the process of using external models such as gpt-35-turbo. We propose a cooperative game dubbed {``}HiddenTables{''} as a potential resolution to this challenge. In essence, {``}HiddenTables{''} is played between the code-generating LLM {``}Solver{''} and the {``}Oracle{''} which evaluates the ability of the LLM agents to solve TableQA tasks. This game is based on natural language schemas and importantly, ensures the security of the underlying data. We provide evidential experiments on a diverse set of tables that demonstrate an LLM{'}s collective inability to generalize and perform on complex queries, handle compositional dependencies, and align natural language to programmatic commands when concrete table schemas are provided. Unlike encoder-based models, we have pushed the boundaries of {``}HiddenTables{''} to not be limited by the number of rows - therefore we exhibit improved efficiency in prompt and completion tokens. Our infrastructure has spawned a new dataset {``}PyQTax{''} that spans across 116,671 question-table-answer triplets and provides additional fine-grained breakdowns and labels for varying question taxonomies. Therefore, in tandem with our academic contributions regarding LLMs{'} deficiency in TableQA tasks, {``}HiddenTables{''} is a tactile manifestation of how LLMs can interact with massive datasets while ensuring data security and minimizing generation costs.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="watson-etal-2023-hiddentables">
<titleInfo>
<title>HiddenTables and PyQTax: A Cooperative Game and Dataset For TableQA to Ensure Scale and Data Privacy Across a Myriad of Taxonomies</title>
</titleInfo>
<name type="personal">
<namePart type="given">William</namePart>
<namePart type="family">Watson</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nicole</namePart>
<namePart type="family">Cho</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tucker</namePart>
<namePart type="family">Balch</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Manuela</namePart>
<namePart type="family">Veloso</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-12</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Houda</namePart>
<namePart type="family">Bouamor</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juan</namePart>
<namePart type="family">Pino</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kalika</namePart>
<namePart type="family">Bali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Singapore</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>A myriad of different Large Language Models (LLMs) face a common challenge in contextually analyzing table question-answering tasks. These challenges are engendered from (1) finite context windows for large tables, (2) multi-faceted discrepancies amongst tokenization patterns against cell boundaries, and (3) various limitations stemming from data confidentiality in the process of using external models such as gpt-35-turbo. We propose a cooperative game dubbed “HiddenTables” as a potential resolution to this challenge. In essence, “HiddenTables” is played between the code-generating LLM “Solver” and the “Oracle” which evaluates the ability of the LLM agents to solve TableQA tasks. This game is based on natural language schemas and importantly, ensures the security of the underlying data. We provide evidential experiments on a diverse set of tables that demonstrate an LLM’s collective inability to generalize and perform on complex queries, handle compositional dependencies, and align natural language to programmatic commands when concrete table schemas are provided. Unlike encoder-based models, we have pushed the boundaries of “HiddenTables” to not be limited by the number of rows - therefore we exhibit improved efficiency in prompt and completion tokens. Our infrastructure has spawned a new dataset “PyQTax” that spans across 116,671 question-table-answer triplets and provides additional fine-grained breakdowns and labels for varying question taxonomies. Therefore, in tandem with our academic contributions regarding LLMs’ deficiency in TableQA tasks, “HiddenTables” is a tactile manifestation of how LLMs can interact with massive datasets while ensuring data security and minimizing generation costs.</abstract>
<identifier type="citekey">watson-etal-2023-hiddentables</identifier>
<identifier type="doi">10.18653/v1/2023.emnlp-main.442</identifier>
<location>
<url>https://aclanthology.org/2023.emnlp-main.442</url>
</location>
<part>
<date>2023-12</date>
<extent unit="page">
<start>7144</start>
<end>7159</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T HiddenTables and PyQTax: A Cooperative Game and Dataset For TableQA to Ensure Scale and Data Privacy Across a Myriad of Taxonomies
%A Watson, William
%A Cho, Nicole
%A Balch, Tucker
%A Veloso, Manuela
%Y Bouamor, Houda
%Y Pino, Juan
%Y Bali, Kalika
%S Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing
%D 2023
%8 December
%I Association for Computational Linguistics
%C Singapore
%F watson-etal-2023-hiddentables
%X A myriad of different Large Language Models (LLMs) face a common challenge in contextually analyzing table question-answering tasks. These challenges are engendered from (1) finite context windows for large tables, (2) multi-faceted discrepancies amongst tokenization patterns against cell boundaries, and (3) various limitations stemming from data confidentiality in the process of using external models such as gpt-35-turbo. We propose a cooperative game dubbed “HiddenTables” as a potential resolution to this challenge. In essence, “HiddenTables” is played between the code-generating LLM “Solver” and the “Oracle” which evaluates the ability of the LLM agents to solve TableQA tasks. This game is based on natural language schemas and importantly, ensures the security of the underlying data. We provide evidential experiments on a diverse set of tables that demonstrate an LLM’s collective inability to generalize and perform on complex queries, handle compositional dependencies, and align natural language to programmatic commands when concrete table schemas are provided. Unlike encoder-based models, we have pushed the boundaries of “HiddenTables” to not be limited by the number of rows - therefore we exhibit improved efficiency in prompt and completion tokens. Our infrastructure has spawned a new dataset “PyQTax” that spans across 116,671 question-table-answer triplets and provides additional fine-grained breakdowns and labels for varying question taxonomies. Therefore, in tandem with our academic contributions regarding LLMs’ deficiency in TableQA tasks, “HiddenTables” is a tactile manifestation of how LLMs can interact with massive datasets while ensuring data security and minimizing generation costs.
%R 10.18653/v1/2023.emnlp-main.442
%U https://aclanthology.org/2023.emnlp-main.442
%U https://doi.org/10.18653/v1/2023.emnlp-main.442
%P 7144-7159
Markdown (Informal)
[HiddenTables and PyQTax: A Cooperative Game and Dataset For TableQA to Ensure Scale and Data Privacy Across a Myriad of Taxonomies](https://aclanthology.org/2023.emnlp-main.442) (Watson et al., EMNLP 2023)
ACL