{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "838818ae-922e-44b4-b7bf-b826932d40d3", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 177, "referenced_widgets": [ "9a228d2d9db54ebbbe9303057707cace", "8f6d41b34fbc40e09336c39dbcd48134", "8ff869a0a89148a1a48fe02c81405fd9", "216ca4a628b740748d4e2a173f2b8696", "e1b37a950a37485bbae7b4a30af76a8f", "05969b99e1ea4ed890281f9893a1a0f0", "f7b71230949a45f7b2ad1eb597bc55f2", "80c3e0a0b2f2413ca19389aad35cb826", "b0c6a5d0b3a8412394805a7283b1a461", "c9b73a8be75e4db182bad129d88a4f56", "bee08a2142c74b5aa0be23716fa9869e", "bd5bb1f7042b465fbb8a4ef450004d08", "b7b7d1d26d8647a18da05f138717a43e", "2097bf04c72a4291bdd37cc454e2b0be", "fc6d819156f44b23a0cb24a69e2e85b6", "375fb84bf0b640f48ed6dc3314dac52a", "03db73b638cf4d76bc252a6713b5d96e", "7b4c06323b29467ca716af2dd7424bd3", "9f6958f5b5de4054950a1b8021c263a5", "6b59835fd7e3447bb1e6c53a6a6403b8", "2836d7aa25e84a589013928428c1d790", "03c6447194294f6da90829911490623d", "ccb092ae57b943a8ba061e5f5011ed61", "efd3141ff1704ce2b09628f70ce874fc", "c2e5fe41424e408a8f6a812a166a96c0", "2dacc6ec36f54fdb853c6ac0bd24794f", "c073089aee4e4921b7c33ba00b59eb36", "1164895bd7424790863e40fa9ddd40a6", "4ff2f4bf20fc45a38414ace547fc1587", "a8ef469b87f94e78a12ed11d4be9d6ef", "65723021ed554cba8299cbe342d27a9f", "2a2cd64eb8df45f9bb5d33b48a5e229e", "66ae3c91f7f440708e5be99c4acc5050", "6c7278cc558349cebf9c1feb3aaef8d0", "dd4123fe41b24bbea3bcbda05f5097f0", "7d5a97bd3f8d4d4683ec05046a926ea8", "3424eee4514c4a19ae689a6b6081efd7", "2c98b26104c4447aa05cfdc4bcd918b2", "5c6d2367e4f947a3a0939395472e9bad", "794ed0446f87482a969a192e241e610c", "f0c824b6aa9141a6bc40c65e51f23aa6", "53f183efe21f45a7816a20eeb9b80a9c", "29fdb93531884d089bee6328aab7d633", "27e7a6a77f7c45508483f1ee0f7676a6", "fec0d915b82a4893bb4551c306a264dc", "a7e70b8baa1e4642b93ea3d6d4bf1915", "07973acb476d4ffc981cf67f01df30d7", "9ccc5a24a0f7419291f611d23e227df4", "32877b6cf0c443eda14e13ae8fca22db", "e90b04c89f2a4d76a9f85dcd33e2a832", "cf5c9db6cfa24fbc905e6cdaf53d4f37", "1fdb1879ff5246aaa2bad88f47a59f06", "16b1ab739e1a43b6b9ad2c65eecaf158", "2f541f3341b24d22b1d77d6add0b396a", "93a63cc64e0045e785806aca853fa589" ] }, "id": "838818ae-922e-44b4-b7bf-b826932d40d3", "outputId": "cd1265c3-335e-41d3-ca0b-8ab75fbdabe2" }, "outputs": [], "source": [ "from datasets import load_dataset\n", "\n", "ds = load_dataset(\"Jinyan1/COLING_2025_MGT_en\") #import dataset from hugging face" ] }, { "cell_type": "code", "execution_count": 17, "id": "0df0eeef-d50a-419d-8ef1-f6ba49a9b4dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dataset({\n", " features: ['id', 'lang', 'text'],\n", " num_rows: 73941\n", "})\n" ] } ], "source": [ "import json\n", "from datasets import Dataset\n", "\n", "# Path to the test file (change this to the actual path on your system)\n", "test_file_path = 'test_set_en (1).jsonl'\n", "\n", "# Read the JSONL file and load IDs\n", "test_data = []\n", "with open(test_file_path, 'r') as file:\n", " for line in file:\n", " data = json.loads(line.strip())\n", " test_data.append(data)\n", "\n", "# Extract IDs and texts based on your JSON structure\n", "# Adjust the keys based on the structure of your JSON\n", "text = [entry.get('text') for entry in test_data] # Use the correct key for text\n", "id = [entry.get('testset_id') for entry in test_data]\n", "lang = [entry.get('lang') for entry in test_data]\n", "\n", "# Create a dictionary for the Dataset\n", "dataset_dict = {\n", " 'id': id,\n", " 'lang':lang,\n", " 'text': text, # Make sure to include the text field\n", "}\n", "\n", "# Create the Hugging Face Dataset\n", "test_dataset = Dataset.from_dict(dataset_dict)\n", "\n", "# Now set ds['test'] to the test dataset\n", "das = {'test': test_dataset}\n", "\n", "# Print the dataset info to verify\n", "print(das['test'])\n" ] }, { "cell_type": "code", "execution_count": 18, "id": "eb85468c-3a7c-48fb-aaa5-875df28a4af6", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "eb85468c-3a7c-48fb-aaa5-875df28a4af6", "outputId": "4c49b68a-19f6-47b8-a31c-129db0cfce54", "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dataset({\n", " features: ['id', 'source', 'sub_source', 'lang', 'model', 'label', 'text'],\n", " num_rows: 610767\n", "})\n", "{'train': ['id', 'source', 'sub_source', 'lang', 'model', 'label', 'text'], 'dev': ['id', 'source', 'sub_source', 'lang', 'model', 'label', 'text']}\n", "{'id': Value(dtype='string', id=None), 'source': Value(dtype='string', id=None), 'sub_source': Value(dtype='string', id=None), 'lang': Value(dtype='string', id=None), 'model': Value(dtype='string', id=None), 'label': Value(dtype='int64', id=None), 'text': Value(dtype='string', id=None)}\n", "{'id': 'f05034ca-d1da-445d-a6a2-5869ade0dfc3', 'source': 'm4gt', 'sub_source': 'reddit', 'lang': 'en', 'model': 'llama3-8b', 'label': 1, 'text': \"Hitler's plans for the succession and power structure after his death are shrouded in mystery, as he never explicitly wrote down his intentions. However, it is known that he designated several potential successors, including Heinrich Himmler, Hermann Göring, and Joseph Goebbels, each with their own strengths and weaknesses. \\n\\nIn the final days of his life, Hitler grew increasingly paranoid and isolated, leading to rumors of a civil war within the Nazi Party. He became convinced that the Party was plotting against him, and his trusted advisors, including Goering and Himmler, were secretly vying for power. \\n\\nTo counter this perceived threat, Hitler reportedly planned to appoint a successor, most likely Himmler, in a 'Fuhrer-proof' mechanism. This would have granted Himmler absolute power over the Party and state, rendering the concept of democratic succession irrelevant. Hitler also envisioned a hierarchical, hereditary power structure, with his loyal followers and their families holding key positions within the Party and government. \\n\\nMoreover, Hitler's plans likely included implementing a system of informal networks and patronage, where senior officials would be incentivized to support his designated successor. This would have allowed him to exercise control from beyond the grave, ensuring that his vision for the thousand-year Reich remained intact. In essence, Hitler aimed to create a personalized, authoritarian regime where power would remain concentrated in the hands of the 'Fuhrer's' progeny, thereby guaranteeing the continuation of his legacy.\"}\n" ] } ], "source": [ "print(ds['train'])\n", "print(ds.column_names)\n", "print(ds[\"train\"].features)\n", "print(ds[\"train\"][0]) # preview dataset" ] }, { "cell_type": "code", "execution_count": 19, "id": "a3f1577a-0c0d-4667-9529-f089aef65ffc", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "a3f1577a-0c0d-4667-9529-f089aef65ffc", "outputId": "66462abf-cc53-4e25-a65b-aa2c73523afd" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: nltk in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (3.9.1)\n", "Requirement already satisfied: click in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nltk) (8.1.7)\n", "Requirement already satisfied: joblib in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nltk) (1.4.2)\n", "Requirement already satisfied: regex>=2021.8.3 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nltk) (2024.9.11)\n", "Requirement already satisfied: tqdm in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nltk) (4.66.5)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package punkt to\n", "[nltk_data] /teamspace/studios/this_studio/nltk_data...\n", "[nltk_data] Package punkt is already up-to-date!\n", "[nltk_data] Downloading package punkt_tab to\n", "[nltk_data] /teamspace/studios/this_studio/nltk_data...\n", "[nltk_data] Package punkt_tab is already up-to-date!\n", "[nltk_data] Downloading package wordnet to\n", "[nltk_data] /teamspace/studios/this_studio/nltk_data...\n", "[nltk_data] Package wordnet is already up-to-date!\n", "[nltk_data] Downloading package omw-1.4 to\n", "[nltk_data] /teamspace/studios/this_studio/nltk_data...\n", "[nltk_data] Package omw-1.4 is already up-to-date!\n" ] } ], "source": [ "!pip install nltk\n", "import nltk\n", "from nltk.corpus import stopwords\n", "from nltk.stem import WordNetLemmatizer\n", "import string\n", "\n", "# Download necessary resources if not already available\n", "nltk.download(\"punkt\")\n", "nltk.download(\"punkt_tab\")\n", "nltk.download(\"wordnet\")\n", "nltk.download(\"omw-1.4\")\n", "\n", "# Initializing Wordnet lemmatizer\n", "lemmatizer = WordNetLemmatizer()\n", "\n", "# Function to clean, tokenize, and lemmatize tokens\n", "def clean_and_tokenize(batch):\n", " # Tokenize the text using NLTK's word_tokenize for better handling of punctuation\n", " tokens = nltk.word_tokenize(batch[\"text\"])\n", "\n", " # Convert to lowercase, and lemmatize tokens\n", " filtered_tokens = [\n", " lemmatizer.lemmatize(token.lower()) for token in tokens\n", " if token.isalpha()\n", " ]\n", "\n", " return {\n", " \"cleaned_text\": \" \".join(filtered_tokens), # Return cleaned and lemmatized text as a string\n", " \"tokens\": filtered_tokens # Return lemmatized tokens as a list\n", " }\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "4522a26e-8a7c-4d0c-8cd9-853924c0040e", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 81, "referenced_widgets": [ "54dced0302624b5b95e7db579708185c", "95fc38182e3d4b9c9f866b06399bc95a", "eca7946e0d6a494d9d879f53041f1dbd", "51e031be603445cf9210d2bb710154a9", "570b70ab08994c3d881df2897cf17fee", "3ae8d0914fa3419b85dd5768bd82f264", "58d3034b60604b87b1ad841023ca1e4d", "f4467759f50345f29ab15525f1cd55cd", "cb892d0571ae49718780f19dddb07c33", "776f6e673a7641819bb0c3ff4a2cc773", "dee3e6b608614446b89d59b978bb9962", "7ed12e310f5943819b992ec477c09b81", "70b57cec926b41c896b24af4cc97a31a", "f8edaf6dcb864edf81914ffacdbe618a", "0d5e6e6977bd4c42a9a3c3663b4cdbf3", "443d270852f74584bb819d1d81258fad", "cb11dc900cfa44f3a19d44352ced5831", "7b6a190520974df18dc5c3a2a8b9ea14", "676854167a444c3e8b1ee4e0e7493dc3", "660620cff95e43fa8e5e4047cabbe3a0", "efd76afd4f5844d5ae1ccde234d21f8e", "f4e1b10d5f654614ae8b09d5784a758f" ] }, "id": "4522a26e-8a7c-4d0c-8cd9-853924c0040e", "outputId": "9242270c-94e1-4000-d264-6331905a8381", "scrolled": true }, "outputs": [], "source": [ "tokenized_dataset = ds.map(clean_and_tokenize, batched=False, num_proc=4)" ] }, { "cell_type": "code", "execution_count": 20, "id": "0a6c6912-04f3-4af6-acb7-7f9ec99e408f", "metadata": { "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "02ed5e04366d4a46bbbf9cd126aec13a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map (num_proc=4): 0%| | 0/73941 [00:00\n", "Traceback (most recent call last):\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1479, in __del__\n", " self._shutdown_workers()\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1462, in _shutdown_workers\n", " if w.is_alive():\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n", " assert self._parent_pid == os.getpid(), 'can only test a child process'\n", "AssertionError: can only test a child process\n", "Exception ignored in: \n", "Traceback (most recent call last):\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1479, in __del__\n", " self._shutdown_workers()\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1462, in _shutdown_workers\n", " if w.is_alive():\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n", " assert self._parent_pid == os.getpid(), 'can only test a child process'\n", "AssertionError: can only test a child process\n", "Exception ignored in: \n", "Traceback (most recent call last):\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1479, in __del__\n", " self._shutdown_workers()\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1462, in _shutdown_workers\n", " if w.is_alive():\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n", " assert self._parent_pid == os.getpid(), 'can only test a child process'\n", "AssertionError: can only test a child process\n", "Exception ignored in: \n", "Traceback (most recent call last):\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1479, in __del__\n", " self._shutdown_workers()\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1462, in _shutdown_workers\n", " if w.is_alive():\n", " File \"/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n", " assert self._parent_pid == os.getpid(), 'can only test a child process'\n", "AssertionError: can only test a child process\n" ] } ], "source": [ "tokenized_testset = das['test'].map(clean_and_tokenize, batched=False, num_proc=4)" ] }, { "cell_type": "code", "execution_count": 7, "id": "18482711-ce6a-46f5-b85a-2affe08037b6", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "18482711-ce6a-46f5-b85a-2affe08037b6", "outputId": "a6b31b68-96ef-4a5f-efd2-75d4a6aa0fec" }, "outputs": [], "source": [ "from transformers import DistilBertTokenizer\n", "from torch.utils.data import Dataset\n", "\n", "# Initialize the tokenizer\n", "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n", "\n", "class CustomDataset(Dataset):\n", " def __init__(self, examples):\n", " self.examples = examples\n", "\n", " def __len__(self):\n", " return len(self.examples)\n", "\n", " def __getitem__(self, idx):\n", " example = self.examples[idx]\n", " # Tokenize the cleaned text\n", " encoding = tokenizer(\n", " example['cleaned_text'], # Using tokens for tokenization\n", " truncation=True,\n", " padding='max_length',\n", " max_length=512,\n", " return_tensors='pt'\n", " )\n", " return {\n", " 'input_ids': encoding['input_ids'].flatten(),\n", " 'attention_mask': encoding['attention_mask'].flatten(),\n", " 'label': example['label'],\n", " 'id': example['id']\n", " }\n", "\n", "\n", "# Create the dataset instances\n", "train_dataset = CustomDataset(tokenized_dataset['train'])\n", "dev_dataset = CustomDataset(tokenized_dataset['dev'])" ] }, { "cell_type": "code", "execution_count": 21, "id": "daa3a00d-ecbe-45b5-a730-492326f38057", "metadata": {}, "outputs": [], "source": [ "from transformers import DistilBertTokenizer\n", "from torch.utils.data import Dataset\n", "\n", "# Initialize the tokenizer\n", "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n", "\n", "class CustomDatasett(Dataset):\n", " def __init__(self, examples):\n", " self.examples = examples\n", "\n", " def __len__(self):\n", " return len(self.examples)\n", "\n", " def __getitem__(self, idx):\n", " example = self.examples[idx]\n", " # Tokenize the cleaned text\n", " encoding = tokenizer(\n", " example['cleaned_text'], # Using tokens for tokenization\n", " truncation=True,\n", " padding='max_length',\n", " max_length=512,\n", " return_tensors='pt'\n", " )\n", " return {\n", " 'input_ids': encoding['input_ids'].flatten(),\n", " 'attention_mask': encoding['attention_mask'].flatten(),\n", " 'id': example['id']\n", " }\n", "\n", "test_dataset = CustomDatasett(tokenized_testset)" ] }, { "cell_type": "code", "execution_count": null, "id": "0b67825a-e74b-44e5-b32a-dc28b7025b7c", "metadata": { "scrolled": true }, "outputs": [], "source": [ "import os\n", "import torch\n", "from transformers import (\n", " DistilBertForSequenceClassification, \n", " DistilBertTokenizer,\n", " Trainer, \n", " TrainingArguments, \n", " DataCollatorWithPadding\n", ")\n", "import evaluate # Use the evaluate library for metrics\n", "\n", "# Check if CUDA is available\n", "print(\"CUDA available:\", torch.cuda.is_available())\n", "if not torch.cuda.is_available():\n", " print(\"Warning: CUDA is not available, training will be done on CPU.\")\n", "\n", "# Load the model and tokenizer\n", "model_name = 'distilbert-base-uncased'\n", "model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=2)\n", "tokenizer = DistilBertTokenizer.from_pretrained(model_name)\n", "\n", "# Load metrics\n", "accuracy_metric = evaluate.load(\"accuracy\")\n", "f1_metric = evaluate.load(\"f1\")\n", "\n", "def compute_metrics(eval_pred):\n", " logits, labels = eval_pred\n", " predictions = logits.argmax(-1)\n", "\n", " accuracy = accuracy_metric.compute(predictions=predictions, references=labels)\n", " f1 = f1_metric.compute(predictions=predictions, references=labels, average=\"weighted\")\n", "\n", " return {\"accuracy\": accuracy[\"accuracy\"], \"f1\": f1[\"f1\"]}\n", "\n", "# Updated TrainingArguments for better utilization\n", "training_args = TrainingArguments(\n", " output_dir='./results',\n", " eval_strategy='epoch', \n", " per_device_train_batch_size=256, \n", " per_device_eval_batch_size=256,\n", " gradient_accumulation_steps=2, \n", " num_train_epochs=3, \n", " weight_decay=0.02,\n", " logging_dir='./logs',\n", " logging_steps=10,\n", " fp16=True, \n", " bf16=False, \n", " dataloader_num_workers=8, \n", " save_strategy=\"epoch\",\n", " load_best_model_at_end=True,\n", " metric_for_best_model=\"f1\", \n", " save_total_limit=2, \n", " lr_scheduler_type=\"cosine\", \n", " warmup_steps=500, \n", " save_steps=1000, \n", ")\n", "\n", "# Initialize data collator for padding\n", "data_collator = DataCollatorWithPadding(tokenizer=tokenizer) # Use the initialized tokenizer\n", "\n", "# Define the Trainer without early stopping\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=train_dataset,\n", " eval_dataset=dev_dataset,\n", " compute_metrics=compute_metrics, \n", " data_collator=data_collator \n", ")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d8e45d8b-3dbc-4b63-9496-51d805b6e896", "metadata": { "id": "d8e45d8b-3dbc-4b63-9496-51d805b6e896", "outputId": "160ab61b-c7ac-4a97-9e7f-776cc77ebb2f" }, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [14316/14316 3:52:09, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossAccuracyF1
10.1975000.1759930.9270130.926395
20.1038000.1509550.9439940.943444
30.0541000.1807250.9466530.946111

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "TrainOutput(global_step=14316, training_loss=0.1400702884761639, metrics={'train_runtime': 13931.6745, 'train_samples_per_second': 131.521, 'train_steps_per_second': 1.028, 'total_flos': 2.4272014702478746e+17, 'train_loss': 0.1400702884761639, 'epoch': 3.0})" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "trainer.train()" ] }, { "cell_type": "code", "execution_count": 23, "id": "Gt4oJTV6kXIr", "metadata": { "id": "Gt4oJTV6kXIr" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Making predictions: 100%|██████████| 4622/4622 [05:57<00:00, 12.94it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Predictions saved to ./results/engraw/predictions.jsonl\n" ] } ], "source": [ "import os\n", "import torch\n", "import json\n", "from torch.utils.data import DataLoader\n", "from tqdm import tqdm # For progress bar\n", "\n", "# Define the device for prediction\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "# Function to make predictions and save them to JSONL\n", "def make_predictions_and_save(model, dataloader, output_file):\n", " model.eval() # Set the model to evaluation mode\n", " model.to(device) # Move the model to the device\n", " all_predictions = []\n", "\n", " with torch.no_grad(): # No need to track gradients\n", " # Add tqdm to the dataloader for progress tracking\n", " for batch in tqdm(dataloader, desc=\"Making predictions\"):\n", " input_ids = batch['input_ids'].to(device)\n", " attention_mask = batch['attention_mask'].to(device)\n", "\n", " # Get predictions\n", " outputs = model(input_ids, attention_mask=attention_mask)\n", " logits = outputs.logits\n", " predictions = torch.argmax(logits, dim=1).cpu().numpy() # Get predicted labels\n", "\n", " # Collect predictions with corresponding IDs\n", " for idx, pred in zip(batch['id'], predictions):\n", " all_predictions.append({\"id\": int(idx), \"label\": int(pred)})\n", "\n", " # Save to JSON Lines format\n", " os.makedirs(os.path.dirname(output_file), exist_ok=True) # Ensure output directory exists\n", " with open(output_file, 'w') as f:\n", " for prediction in all_predictions:\n", " f.write(json.dumps(prediction) + '\\n')\n", "\n", " print(f\"Predictions saved to {output_file}\")\n", "\n", "# Create a DataLoader for the test dataset\n", "dev_dataloader = DataLoader(test_dataset, batch_size=16)\n", "\n", "# After training, make predictions and save them to JSONL\n", "output_file_path = './results/engraw/predictions.jsonl' # Adjust path as necessary\n", "make_predictions_and_save(trainer.model, dev_dataloader, output_file_path)\n" ] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.10" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "03c6447194294f6da90829911490623d": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "03db73b638cf4d76bc252a6713b5d96e": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "05969b99e1ea4ed890281f9893a1a0f0": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "07973acb476d4ffc981cf67f01df30d7": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1fdb1879ff5246aaa2bad88f47a59f06", "max": 261758, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_16b1ab739e1a43b6b9ad2c65eecaf158", "value": 261758 } }, "0d5e6e6977bd4c42a9a3c3663b4cdbf3": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_efd76afd4f5844d5ae1ccde234d21f8e", "placeholder": "​", "style": "IPY_MODEL_f4e1b10d5f654614ae8b09d5784a758f", "value": " 261758/261758 [16:22<00:00, 196.37 examples/s]" } }, "0db54c2d21fd4c709e906b10d90d8f43": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_6ae6547971e34732bf40f38a3d3b4242", "IPY_MODEL_64e3919cc7f04790a5e13201a5431e2e", "IPY_MODEL_b60c35ffc82c455f8b1d28e08ccff161" ], "layout": "IPY_MODEL_cee83297c1ee4b229eb6115ef7e64c16" } }, "1164895bd7424790863e40fa9ddd40a6": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "16b1ab739e1a43b6b9ad2c65eecaf158": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "1858d111c0fb4e8b8d4663db53d71dc5": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1f9fddf732e140429079cf956a8ea63d": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1fdb1879ff5246aaa2bad88f47a59f06": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2097bf04c72a4291bdd37cc454e2b0be": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9f6958f5b5de4054950a1b8021c263a5", "max": 286346762, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_6b59835fd7e3447bb1e6c53a6a6403b8", "value": 286346762 } }, "216ca4a628b740748d4e2a173f2b8696": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c9b73a8be75e4db182bad129d88a4f56", "placeholder": "​", "style": "IPY_MODEL_bee08a2142c74b5aa0be23716fa9869e", "value": " 287M/287M [00:06<00:00, 42.5MB/s]" } }, "26087f73b9e34891845425376fe10a7f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "27e7a6a77f7c45508483f1ee0f7676a6": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "2836d7aa25e84a589013928428c1d790": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "29fdb93531884d089bee6328aab7d633": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2a2cd64eb8df45f9bb5d33b48a5e229e": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2c98b26104c4447aa05cfdc4bcd918b2": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2dacc6ec36f54fdb853c6ac0bd24794f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2a2cd64eb8df45f9bb5d33b48a5e229e", "placeholder": "​", "style": "IPY_MODEL_66ae3c91f7f440708e5be99c4acc5050", "value": " 246M/246M [00:05<00:00, 42.6MB/s]" } }, "2f541f3341b24d22b1d77d6add0b396a": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "32877b6cf0c443eda14e13ae8fca22db": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3424eee4514c4a19ae689a6b6081efd7": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_29fdb93531884d089bee6328aab7d633", "placeholder": "​", "style": "IPY_MODEL_27e7a6a77f7c45508483f1ee0f7676a6", "value": " 610767/610767 [00:07<00:00, 96115.98 examples/s]" } }, "375fb84bf0b640f48ed6dc3314dac52a": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3ae8d0914fa3419b85dd5768bd82f264": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3e740f4069bf4790a0febb870005190f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1f9fddf732e140429079cf956a8ea63d", "placeholder": "​", "style": "IPY_MODEL_ee2fd52564b8444ca2b13ec2bc23a47b", "value": "config.json: 100%" } }, "443d270852f74584bb819d1d81258fad": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4d9e9f027a4342c0954d7c8c8398812f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "4ff2f4bf20fc45a38414ace547fc1587": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "51e031be603445cf9210d2bb710154a9": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_776f6e673a7641819bb0c3ff4a2cc773", "placeholder": "​", "style": "IPY_MODEL_dee3e6b608614446b89d59b978bb9962", "value": " 610767/610767 [38:18<00:00, 377.79 examples/s]" } }, "53f183efe21f45a7816a20eeb9b80a9c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "54dced0302624b5b95e7db579708185c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_95fc38182e3d4b9c9f866b06399bc95a", "IPY_MODEL_eca7946e0d6a494d9d879f53041f1dbd", "IPY_MODEL_51e031be603445cf9210d2bb710154a9" ], "layout": "IPY_MODEL_570b70ab08994c3d881df2897cf17fee" } }, "570b70ab08994c3d881df2897cf17fee": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "577827e198d64f5c844c941358db9243": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "58d3034b60604b87b1ad841023ca1e4d": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5c6d2367e4f947a3a0939395472e9bad": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "64b13d46af574ddfa9f596ed5a67ae56": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "64e3919cc7f04790a5e13201a5431e2e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c1a121874bef4ac3b110cc8576fed5ec", "max": 267954768, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_a3d98f3cdc524e7c94b14c7e2583f06e", "value": 267954768 } }, "65723021ed554cba8299cbe342d27a9f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "660620cff95e43fa8e5e4047cabbe3a0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "668b3d494793451db7dc590d168af703": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "66ae3c91f7f440708e5be99c4acc5050": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "676854167a444c3e8b1ee4e0e7493dc3": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6ae6547971e34732bf40f38a3d3b4242": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_668b3d494793451db7dc590d168af703", "placeholder": "​", "style": "IPY_MODEL_26087f73b9e34891845425376fe10a7f", "value": "model.safetensors: 100%" } }, "6b59835fd7e3447bb1e6c53a6a6403b8": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "6c7278cc558349cebf9c1feb3aaef8d0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_dd4123fe41b24bbea3bcbda05f5097f0", "IPY_MODEL_7d5a97bd3f8d4d4683ec05046a926ea8", "IPY_MODEL_3424eee4514c4a19ae689a6b6081efd7" ], "layout": "IPY_MODEL_2c98b26104c4447aa05cfdc4bcd918b2" } }, "70b57cec926b41c896b24af4cc97a31a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cb11dc900cfa44f3a19d44352ced5831", "placeholder": "​", "style": "IPY_MODEL_7b6a190520974df18dc5c3a2a8b9ea14", "value": "Map (num_proc=8): 100%" } }, "776f6e673a7641819bb0c3ff4a2cc773": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "794ed0446f87482a969a192e241e610c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7b4c06323b29467ca716af2dd7424bd3": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7b6a190520974df18dc5c3a2a8b9ea14": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7c89622cabbd4d10b6ff38ca01cf41ee": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7d5a97bd3f8d4d4683ec05046a926ea8": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f0c824b6aa9141a6bc40c65e51f23aa6", "max": 610767, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_53f183efe21f45a7816a20eeb9b80a9c", "value": 610767 } }, "7ed12e310f5943819b992ec477c09b81": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_70b57cec926b41c896b24af4cc97a31a", "IPY_MODEL_f8edaf6dcb864edf81914ffacdbe618a", "IPY_MODEL_0d5e6e6977bd4c42a9a3c3663b4cdbf3" ], "layout": "IPY_MODEL_443d270852f74584bb819d1d81258fad" } }, "80c3e0a0b2f2413ca19389aad35cb826": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8f6d41b34fbc40e09336c39dbcd48134": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_05969b99e1ea4ed890281f9893a1a0f0", "placeholder": "​", "style": "IPY_MODEL_f7b71230949a45f7b2ad1eb597bc55f2", "value": "train-00000-of-00002.parquet: 100%" } }, "8ff869a0a89148a1a48fe02c81405fd9": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_80c3e0a0b2f2413ca19389aad35cb826", "max": 287167402, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_b0c6a5d0b3a8412394805a7283b1a461", "value": 287167402 } }, "93a63cc64e0045e785806aca853fa589": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "95fc38182e3d4b9c9f866b06399bc95a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3ae8d0914fa3419b85dd5768bd82f264", "placeholder": "​", "style": "IPY_MODEL_58d3034b60604b87b1ad841023ca1e4d", "value": "Map (num_proc=8): 100%" } }, "9a228d2d9db54ebbbe9303057707cace": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_8f6d41b34fbc40e09336c39dbcd48134", "IPY_MODEL_8ff869a0a89148a1a48fe02c81405fd9", "IPY_MODEL_216ca4a628b740748d4e2a173f2b8696" ], "layout": "IPY_MODEL_e1b37a950a37485bbae7b4a30af76a8f" } }, "9ccc5a24a0f7419291f611d23e227df4": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2f541f3341b24d22b1d77d6add0b396a", "placeholder": "​", "style": "IPY_MODEL_93a63cc64e0045e785806aca853fa589", "value": " 261758/261758 [00:05<00:00, 50006.62 examples/s]" } }, "9f6958f5b5de4054950a1b8021c263a5": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a3d98f3cdc524e7c94b14c7e2583f06e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "a7e70b8baa1e4642b93ea3d6d4bf1915": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e90b04c89f2a4d76a9f85dcd33e2a832", "placeholder": "​", "style": "IPY_MODEL_cf5c9db6cfa24fbc905e6cdaf53d4f37", "value": "Generating dev split: 100%" } }, "a8ef469b87f94e78a12ed11d4be9d6ef": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b0c6a5d0b3a8412394805a7283b1a461": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "b59ca14e198b4018a2911ff89b0eefdf": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e736d6126cc14e4faa665941e9f5b9a8", "placeholder": "​", "style": "IPY_MODEL_4d9e9f027a4342c0954d7c8c8398812f", "value": " 483/483 [00:00<00:00, 14.1kB/s]" } }, "b60c35ffc82c455f8b1d28e08ccff161": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cb71031e59154c8690a5831a29a69652", "placeholder": "​", "style": "IPY_MODEL_1858d111c0fb4e8b8d4663db53d71dc5", "value": " 268M/268M [00:02<00:00, 117MB/s]" } }, "b7b7d1d26d8647a18da05f138717a43e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_03db73b638cf4d76bc252a6713b5d96e", "placeholder": "​", "style": "IPY_MODEL_7b4c06323b29467ca716af2dd7424bd3", "value": "train-00001-of-00002.parquet: 100%" } }, "bd5bb1f7042b465fbb8a4ef450004d08": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_b7b7d1d26d8647a18da05f138717a43e", "IPY_MODEL_2097bf04c72a4291bdd37cc454e2b0be", "IPY_MODEL_fc6d819156f44b23a0cb24a69e2e85b6" ], "layout": "IPY_MODEL_375fb84bf0b640f48ed6dc3314dac52a" } }, "bee08a2142c74b5aa0be23716fa9869e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "c073089aee4e4921b7c33ba00b59eb36": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c1a121874bef4ac3b110cc8576fed5ec": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c2e5fe41424e408a8f6a812a166a96c0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a8ef469b87f94e78a12ed11d4be9d6ef", "max": 245872054, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_65723021ed554cba8299cbe342d27a9f", "value": 245872054 } }, "c9b73a8be75e4db182bad129d88a4f56": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "cb11dc900cfa44f3a19d44352ced5831": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "cb71031e59154c8690a5831a29a69652": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "cb892d0571ae49718780f19dddb07c33": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "ccb092ae57b943a8ba061e5f5011ed61": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_efd3141ff1704ce2b09628f70ce874fc", "IPY_MODEL_c2e5fe41424e408a8f6a812a166a96c0", "IPY_MODEL_2dacc6ec36f54fdb853c6ac0bd24794f" ], "layout": "IPY_MODEL_c073089aee4e4921b7c33ba00b59eb36" } }, "cee83297c1ee4b229eb6115ef7e64c16": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "cf5c9db6cfa24fbc905e6cdaf53d4f37": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d6967a43a4d14cbd8d58a2dd04bb96ee": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_3e740f4069bf4790a0febb870005190f", "IPY_MODEL_f437c6cd246b42c1ba2ca82cce2c45eb", "IPY_MODEL_b59ca14e198b4018a2911ff89b0eefdf" ], "layout": "IPY_MODEL_7c89622cabbd4d10b6ff38ca01cf41ee" } }, "dd4123fe41b24bbea3bcbda05f5097f0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_5c6d2367e4f947a3a0939395472e9bad", "placeholder": "​", "style": "IPY_MODEL_794ed0446f87482a969a192e241e610c", "value": "Generating train split: 100%" } }, "dee3e6b608614446b89d59b978bb9962": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e1b37a950a37485bbae7b4a30af76a8f": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e736d6126cc14e4faa665941e9f5b9a8": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e90b04c89f2a4d76a9f85dcd33e2a832": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "eca7946e0d6a494d9d879f53041f1dbd": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f4467759f50345f29ab15525f1cd55cd", "max": 610767, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_cb892d0571ae49718780f19dddb07c33", "value": 610767 } }, "ee2fd52564b8444ca2b13ec2bc23a47b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "efd3141ff1704ce2b09628f70ce874fc": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1164895bd7424790863e40fa9ddd40a6", "placeholder": "​", "style": "IPY_MODEL_4ff2f4bf20fc45a38414ace547fc1587", "value": "dev-00000-of-00001.parquet: 100%" } }, "efd76afd4f5844d5ae1ccde234d21f8e": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f0c824b6aa9141a6bc40c65e51f23aa6": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f437c6cd246b42c1ba2ca82cce2c45eb": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_577827e198d64f5c844c941358db9243", "max": 483, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_64b13d46af574ddfa9f596ed5a67ae56", "value": 483 } }, "f4467759f50345f29ab15525f1cd55cd": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f4e1b10d5f654614ae8b09d5784a758f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f7b71230949a45f7b2ad1eb597bc55f2": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f8edaf6dcb864edf81914ffacdbe618a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_676854167a444c3e8b1ee4e0e7493dc3", "max": 261758, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_660620cff95e43fa8e5e4047cabbe3a0", "value": 261758 } }, "fc6d819156f44b23a0cb24a69e2e85b6": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2836d7aa25e84a589013928428c1d790", "placeholder": "​", "style": "IPY_MODEL_03c6447194294f6da90829911490623d", "value": " 286M/286M [00:07<00:00, 41.6MB/s]" } }, "fec0d915b82a4893bb4551c306a264dc": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a7e70b8baa1e4642b93ea3d6d4bf1915", "IPY_MODEL_07973acb476d4ffc981cf67f01df30d7", "IPY_MODEL_9ccc5a24a0f7419291f611d23e227df4" ], "layout": "IPY_MODEL_32877b6cf0c443eda14e13ae8fca22db" } } } } }, "nbformat": 4, "nbformat_minor": 5 }