22057081_Chandra _shekhar_mahto.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "y98pkyvL-FcE",
    "outputId": "422e6bbd-829d-46ca-f61d-db69e0db8f26"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.38.2)\n",
      "Collecting transformers\n",
      "  Downloading transformers-4.39.1-py3-none-any.whl (8.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.8/8.8 MB\u001b[0m \u001b[31m18.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting datasets[s3]==2.18.0\n",
      "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m31.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting sagemaker>=2.190.0\n",
      "  Downloading sagemaker-2.214.1-py3-none-any.whl (1.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m36.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: huggingface_hub[cli] in /usr/local/lib/python3.10/dist-packages (0.20.3)\n",
      "Collecting huggingface_hub[cli]\n",
      "  Downloading huggingface_hub-0.22.1-py3-none-any.whl (388 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.6/388.6 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (3.13.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (1.25.2)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (14.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (0.6)\n",
      "Collecting dill<0.3.9,>=0.3.0 (from datasets[s3]==2.18.0)\n",
      "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (1.5.3)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (4.66.2)\n",
      "Collecting xxhash (from datasets[s3]==2.18.0)\n",
      "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting multiprocess (from datasets[s3]==2.18.0)\n",
      "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (2023.6.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (3.9.3)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (24.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets[s3]==2.18.0) (6.0.1)\n",
      "Collecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2024.3.1-py3-none-any.whl (29 kB)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.2)\n",
      "Requirement already satisfied: attrs<24,>=23.1.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (23.2.0)\n",
      "Collecting boto3<2.0,>=1.33.3 (from sagemaker>=2.190.0)\n",
      "  Downloading boto3-1.34.72-py3-none-any.whl (139 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.3/139.3 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: cloudpickle==2.2.1 in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (0.2.0)\n",
      "Requirement already satisfied: protobuf<5.0,>=3.12 in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (3.20.3)\n",
      "Collecting smdebug-rulesconfig==1.0.1 (from sagemaker>=2.190.0)\n",
      "  Downloading smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl (20 kB)\n",
      "Collecting importlib-metadata<7.0,>=1.4.0 (from sagemaker>=2.190.0)\n",
      "  Downloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n",
      "Collecting pathos (from sagemaker>=2.190.0)\n",
      "  Downloading pathos-0.3.2-py3-none-any.whl (82 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting schema (from sagemaker>=2.190.0)\n",
      "  Downloading schema-0.7.5-py2.py3-none-any.whl (17 kB)\n",
      "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (4.19.2)\n",
      "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (4.2.0)\n",
      "Requirement already satisfied: tblib<4,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (3.0.0)\n",
      "Requirement already satisfied: urllib3<3.0.0,>=1.26.8 in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (2.0.7)\n",
      "Collecting docker (from sagemaker>=2.190.0)\n",
      "  Downloading docker-7.0.0-py3-none-any.whl (147 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.6/147.6 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from sagemaker>=2.190.0) (5.9.5)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub[cli]) (4.10.0)\n",
      "Collecting InquirerPy==0.3.4 (from huggingface_hub[cli])\n",
      "  Downloading InquirerPy-0.3.4-py3-none-any.whl (67 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.7/67.7 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pfzy<0.4.0,>=0.3.1 (from InquirerPy==0.3.4->huggingface_hub[cli])\n",
      "  Downloading pfzy-0.3.4-py3-none-any.whl (8.5 kB)\n",
      "Requirement already satisfied: prompt-toolkit<4.0.0,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from InquirerPy==0.3.4->huggingface_hub[cli]) (3.0.43)\n",
      "Collecting botocore<1.35.0,>=1.34.72 (from boto3<2.0,>=1.33.3->sagemaker>=2.190.0)\n",
      "  Downloading botocore-1.34.72-py3-none-any.whl (12.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.0/12.0 MB\u001b[0m \u001b[31m40.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3<2.0,>=1.33.3->sagemaker>=2.190.0)\n",
      "  Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
      "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3<2.0,>=1.33.3->sagemaker>=2.190.0)\n",
      "  Downloading s3transfer-0.10.1-py3-none-any.whl (82 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.2/82.2 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets[s3]==2.18.0) (1.3.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets[s3]==2.18.0) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets[s3]==2.18.0) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets[s3]==2.18.0) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets[s3]==2.18.0) (4.0.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker>=2.190.0) (3.18.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets[s3]==2.18.0) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets[s3]==2.18.0) (3.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets[s3]==2.18.0) (2024.2.2)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from google-pasta->sagemaker>=2.190.0) (1.16.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker>=2.190.0) (2023.12.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker>=2.190.0) (0.34.0)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker>=2.190.0) (0.18.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets[s3]==2.18.0) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets[s3]==2.18.0) (2023.4)\n",
      "Collecting ppft>=1.7.6.8 (from pathos->sagemaker>=2.190.0)\n",
      "  Downloading ppft-1.7.6.8-py3-none-any.whl (56 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pox>=0.3.4 (from pathos->sagemaker>=2.190.0)\n",
      "  Downloading pox-0.3.4-py3-none-any.whl (29 kB)\n",
      "Collecting aiobotocore<3.0.0,>=2.5.4 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.12.1-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.3/76.3 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of s3fs to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2024.3.0-py3-none-any.whl (29 kB)\n",
      "  Downloading s3fs-2024.2.0-py3-none-any.whl (28 kB)\n",
      "  Downloading s3fs-2023.12.2-py3-none-any.whl (28 kB)\n",
      "  Downloading s3fs-2023.12.1-py3-none-any.whl (28 kB)\n",
      "  Downloading s3fs-2023.10.0-py3-none-any.whl (28 kB)\n",
      "Collecting aiobotocore~=2.7.0 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.7.0-py3-none-any.whl (73 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.5/73.5 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2023.9.2-py3-none-any.whl (28 kB)\n",
      "Collecting aiobotocore~=2.5.4 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.5.4-py3-none-any.whl (73 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.4/73.4 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2023.9.1-py3-none-any.whl (28 kB)\n",
      "INFO: pip is looking at multiple versions of s3fs to determine which version is compatible with other requirements. This could take a while.\n",
      "  Downloading s3fs-2023.9.0-py3-none-any.whl (28 kB)\n",
      "  Downloading s3fs-2023.6.0-py3-none-any.whl (28 kB)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /usr/local/lib/python3.10/dist-packages (from schema->sagemaker>=2.190.0) (21.6.0)\n",
      "INFO: pip is looking at multiple versions of aiobotocore to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting aiobotocore~=2.5.0 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.5.3-py3-none-any.whl (73 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.3/73.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.5.2-py3-none-any.whl (72 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.9/72.9 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.5.1-py3-none-any.whl (72 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.8/72.8 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.5.0-py3-none-any.whl (72 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.7/72.7 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
      "Collecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2023.5.0-py3-none-any.whl (28 kB)\n",
      "  Downloading s3fs-2023.4.0-py3-none-any.whl (28 kB)\n",
      "  Downloading s3fs-2023.3.0-py3-none-any.whl (27 kB)\n",
      "Collecting aiobotocore~=2.4.2 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.4.2-py3-none-any.whl (66 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.8/66.8 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2023.1.0-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.11.0-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.10.0-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.8.2-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.8.1-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.8.0-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.7.1-py3-none-any.whl (27 kB)\n",
      "Collecting aiobotocore~=2.3.4 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.3.4-py3-none-any.whl (64 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.7/64.7 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2022.7.0-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.5.0-py3-none-any.whl (27 kB)\n",
      "  Downloading s3fs-2022.3.0-py3-none-any.whl (26 kB)\n",
      "Collecting aiobotocore~=2.2.0 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.2.0.tar.gz (59 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.7/59.7 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Collecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2022.2.0-py3-none-any.whl (26 kB)\n",
      "Collecting aiobotocore~=2.1.0 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.1.2-py3-none-any.whl (55 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2022.1.0-py3-none-any.whl (25 kB)\n",
      "  Downloading s3fs-2021.11.1-py3-none-any.whl (25 kB)\n",
      "Collecting aiobotocore~=2.0.1 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.0.1.tar.gz (54 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Collecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2021.11.0-py3-none-any.whl (25 kB)\n",
      "Collecting aiobotocore~=1.4.1 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-1.4.2.tar.gz (52 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Collecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-2021.10.1-py3-none-any.whl (26 kB)\n",
      "  Downloading s3fs-2021.10.0-py3-none-any.whl (26 kB)\n",
      "  Downloading s3fs-2021.9.0-py3-none-any.whl (26 kB)\n",
      "  Downloading s3fs-2021.8.1-py3-none-any.whl (26 kB)\n",
      "  Downloading s3fs-2021.8.0-py3-none-any.whl (26 kB)\n",
      "  Downloading s3fs-2021.7.0-py3-none-any.whl (25 kB)\n",
      "  Downloading s3fs-2021.6.1-py3-none-any.whl (25 kB)\n",
      "  Downloading s3fs-2021.6.0-py3-none-any.whl (24 kB)\n",
      "  Downloading s3fs-2021.5.0-py3-none-any.whl (24 kB)\n",
      "  Downloading s3fs-2021.4.0-py3-none-any.whl (23 kB)\n",
      "  Downloading s3fs-0.6.0-py3-none-any.whl (23 kB)\n",
      "Collecting aiobotocore>=1.0.1 (from s3fs->datasets[s3]==2.18.0)\n",
      "  Downloading aiobotocore-2.12.0-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.1/76.1 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.11.2-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.1/76.1 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of aiobotocore to determine which version is compatible with other requirements. This could take a while.\n",
      "  Downloading aiobotocore-2.11.1-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.1/76.1 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.11.0-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.1/76.1 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.10.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.9.1-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.8/75.8 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.9.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.0/76.0 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
      "  Downloading aiobotocore-2.8.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.0/75.0 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.6.0-py3-none-any.whl (73 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.4/73.4 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.4.1-py3-none-any.whl (66 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.8/66.8 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.4.0-py3-none-any.whl (65 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.8/65.8 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-2.3.3.tar.gz (65 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.7/65.7 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-2.3.2.tar.gz (104 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.8/104.8 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-2.3.1.tar.gz (65 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.3/65.3 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-2.3.0.tar.gz (65 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.1/65.1 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-2.1.1.tar.gz (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-2.1.0.tar.gz (54 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-2.0.0.tar.gz (52 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.0/53.0 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.4.1.tar.gz (52 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.3/52.3 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.4.0.tar.gz (51 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.6/51.6 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.3.3.tar.gz (50 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.6/50.6 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.3.2.tar.gz (49 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.3.1.tar.gz (48 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.8/48.8 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.3.0.tar.gz (48 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.2/48.2 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.2.2.tar.gz (48 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.1/48.1 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.2.1.tar.gz (48 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.0/48.0 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.2.0.tar.gz (47 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.3/47.3 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Downloading aiobotocore-1.1.2-py3-none-any.whl (45 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.1/45.1 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.1.1-py3-none-any.whl (45 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.0/45.0 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.1.0-py3-none-any.whl (43 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.7-py3-none-any.whl (42 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.9/42.9 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.6-py3-none-any.whl (42 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.5-py3-none-any.whl (42 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.4-py3-none-any.whl (41 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.3-py3-none-any.whl (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.2-py3-none-any.whl (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading aiobotocore-1.0.1-py3-none-any.whl (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.7/40.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting s3fs (from datasets[s3]==2.18.0)\n",
      "  Downloading s3fs-0.5.2-py3-none-any.whl (22 kB)\n",
      "  Downloading s3fs-0.5.1-py3-none-any.whl (21 kB)\n",
      "  Downloading s3fs-0.5.0-py3-none-any.whl (21 kB)\n",
      "  Downloading s3fs-0.4.2-py3-none-any.whl (19 kB)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit<4.0.0,>=3.0.1->InquirerPy==0.3.4->huggingface_hub[cli]) (0.2.13)\n",
      "Installing collected packages: xxhash, smdebug-rulesconfig, schema, ppft, pox, pfzy, jmespath, importlib-metadata, dill, multiprocess, InquirerPy, huggingface_hub, docker, botocore, s3transfer, s3fs, pathos, transformers, datasets, boto3, sagemaker\n",
      "  Attempting uninstall: importlib-metadata\n",
      "    Found existing installation: importlib_metadata 7.1.0\n",
      "    Uninstalling importlib_metadata-7.1.0:\n",
      "      Successfully uninstalled importlib_metadata-7.1.0\n",
      "  Attempting uninstall: huggingface_hub\n",
      "    Found existing installation: huggingface-hub 0.20.3\n",
      "    Uninstalling huggingface-hub-0.20.3:\n",
      "      Successfully uninstalled huggingface-hub-0.20.3\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.38.2\n",
      "    Uninstalling transformers-4.38.2:\n",
      "      Successfully uninstalled transformers-4.38.2\n",
      "Successfully installed InquirerPy-0.3.4 boto3-1.34.72 botocore-1.34.72 datasets-2.18.0 dill-0.3.8 docker-7.0.0 huggingface_hub-0.22.1 importlib-metadata-6.11.0 jmespath-1.0.1 multiprocess-0.70.16 pathos-0.3.2 pfzy-0.3.4 pox-0.3.4 ppft-1.7.6.8 s3fs-0.4.2 s3transfer-0.10.1 sagemaker-2.214.1 schema-0.7.5 smdebug-rulesconfig-1.0.1 transformers-4.39.1 xxhash-3.4.1\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers \"datasets[s3]==2.18.0\" \"sagemaker>=2.190.0\" \"huggingface_hub[cli]\" --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yySur09WCKNt",
    "outputId": "cad1e3af-c018-49a7-f0aa-e7e3a94278dc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token has not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
      "Token is valid (permission: read).\n",
      "Your token has been saved to /root/.cache/huggingface/token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "!huggingface-cli login --token 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"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tnVxx56wLypQ",
    "outputId": "dbc708b0-e72c-4a6c-c3af-9de5a485e268"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sagemaker in /usr/local/lib/python3.10/dist-packages (2.214.1)\n",
      "Requirement already satisfied: attrs<24,>=23.1.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (23.2.0)\n",
      "Requirement already satisfied: boto3<2.0,>=1.33.3 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (1.34.72)\n",
      "Requirement already satisfied: cloudpickle==2.2.1 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /usr/local/lib/python3.10/dist-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (1.25.2)\n",
      "Requirement already satisfied: protobuf<5.0,>=3.12 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (3.20.3)\n",
      "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<7.0,>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (6.11.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (24.0)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from sagemaker) (1.5.3)\n",
      "Requirement already satisfied: pathos in /usr/local/lib/python3.10/dist-packages (from sagemaker) (0.3.2)\n",
      "Requirement already satisfied: schema in /usr/local/lib/python3.10/dist-packages (from sagemaker) (0.7.5)\n",
      "Requirement already satisfied: PyYAML~=6.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (6.0.1)\n",
      "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from sagemaker) (4.19.2)\n",
      "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from sagemaker) (4.2.0)\n",
      "Requirement already satisfied: tblib<4,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (3.0.0)\n",
      "Requirement already satisfied: urllib3<3.0.0,>=1.26.8 in /usr/local/lib/python3.10/dist-packages (from sagemaker) (2.0.7)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from sagemaker) (2.31.0)\n",
      "Requirement already satisfied: docker in /usr/local/lib/python3.10/dist-packages (from sagemaker) (7.0.0)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sagemaker) (4.66.2)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from sagemaker) (5.9.5)\n",
      "Requirement already satisfied: botocore<1.35.0,>=1.34.72 in /usr/local/lib/python3.10/dist-packages (from boto3<2.0,>=1.33.3->sagemaker) (1.34.72)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3<2.0,>=1.33.3->sagemaker) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3<2.0,>=1.33.3->sagemaker) (0.10.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker) (3.18.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->sagemaker) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->sagemaker) (3.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->sagemaker) (2024.2.2)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from google-pasta->sagemaker) (1.16.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker) (2023.12.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker) (0.34.0)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker) (0.18.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->sagemaker) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->sagemaker) (2023.4)\n",
      "Requirement already satisfied: ppft>=1.7.6.8 in /usr/local/lib/python3.10/dist-packages (from pathos->sagemaker) (1.7.6.8)\n",
      "Requirement already satisfied: dill>=0.3.8 in /usr/local/lib/python3.10/dist-packages (from pathos->sagemaker) (0.3.8)\n",
      "Requirement already satisfied: pox>=0.3.4 in /usr/local/lib/python3.10/dist-packages (from pathos->sagemaker) (0.3.4)\n",
      "Requirement already satisfied: multiprocess>=0.70.16 in /usr/local/lib/python3.10/dist-packages (from pathos->sagemaker) (0.70.16)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /usr/local/lib/python3.10/dist-packages (from schema->sagemaker) (21.6.0)\n"
     ]
    }
   ],
   "source": [
    "!aws configure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lOY1nUSJFxP6",
    "outputId": "544aa020-84f1-48f0-ef2c-a42dfa56bf1b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:sagemaker:Couldn't call 'get_role' to get Role ARN from role name collab to get Role path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker role arn: arn:aws:iam::381491942612:role/SageMakerTest\n",
      "sagemaker bucket: sagemaker-ap-southeast-2-381491942612\n",
      "sagemaker session region: ap-southeast-2\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "import boto3\n",
    "sess = sagemaker.Session()\n",
    "# sagemaker session bucket -> used for uploading data, models and logs\n",
    "# sagemaker will automatically create this bucket if it not exists\n",
    "sagemaker_session_bucket=None\n",
    "if sagemaker_session_bucket is None and sess is not None:\n",
    "    # set to default bucket if a bucket name is not given\n",
    "    sagemaker_session_bucket = sess.default_bucket()\n",
    "\n",
    "try:\n",
    "    role = sagemaker.get_execution_role()\n",
    "except ValueError:\n",
    "    iam = boto3.client('iam')\n",
    "    role = iam.get_role(RoleName='SageMakerTest')['Role']['Arn']\n",
    "\n",
    "sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)\n",
    "\n",
    "print(f\"sagemaker role arn: {role}\")\n",
    "print(f\"sagemaker bucket: {sess.default_bucket()}\")\n",
    "print(f\"sagemaker session region: {sess.boto_region_name}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 182,
     "referenced_widgets": [
      "7b46ed1fb9c24fefa419ccb433abadab",
      "c56f34bd458441299bf81140b4cfb972",
      "a3c66a59194b4b2899b98efe4d4eb2f8",
      "ba7e2206c665461496e40357e41f41d1",
      "4febf57d3514478dbf0a5294f633e442",
      "019bbb311b6b4f5fa5d50dc18b19e0bf",
      "e4cd0e357d0d4dcc9d58fa3db0dff299",
      "a13603c15ce5469c90b7c65ff9105b81",
      "0765ed8920494d148eaaf164a9c2636b",
      "17d04efd0ebd482698e4ee35ce37b26b",
      "ac9a8ac5f9f642e883de07c3c4696e36",
      "7f8522589dec4c9482404efe2c2bc25b",
      "7ba369d41893482fac304c9ac3775d8d",
      "fd313d073c8847b9bb747a9f27ca9cc0",
      "a2d212ca60fb478e837912c1d8e8635b",
      "6e7233b59b0c4a46b99deeea81380319",
      "fdd46bf54caa463cb296a39ae0d9b663",
      "6741c2fc6cd7458aa7da7e96d4f46516",
      "724ccb94d17840a49441b02be4e6a374",
      "d46dc1ef87ca4297b9c47c7eacdfb0d4",
      "d8e9a52f3b784e30a6e2a9d0a1208caf",
      "a2210d478b1c4782b11745884e9240d6",
      "bfeaaab2c1904277be9127ec8b18f87a",
      "028ce9a6bf8a4633b04f35eac230fd40",
      "a20c87d57f1b4c7288a8dfea6294f857",
      "e751748cd25b407b9a4dded11b168554",
      "4ec195abd6f74d20bb14eb29280d01ff",
      "40503eed415d4b00a8bf4a7fba52d43b",
      "86769be2ea6544488b576b2198b49b34",
      "9b842af5381f47a897498d4304e8e4f3",
      "035962d8eacb4a7fa8bffdc85b511a33",
      "880614c09f7e429e9ce2257d2e3ab068",
      "4119bcd74693409ba223a0b14a1bfcc1",
      "9a90c1a09428458db59b0c27f858d1aa",
      "a2e1ea30d77e40fbaaac91c9fcee9671",
      "7453c7e0b8c046a699283cab05cfa934",
      "e066643b5c40437baf6bdfe00960f210",
      "6464e81f20404c91b5d263d0b09d9d00",
      "b433fce091d143f782f9a21850bc9427",
      "6e2ad038f7c54a8995b071d203ae33a6",
      "c99e46458fa14bbea865e7cc50455d8c",
      "e9d262277bd4409d8d4cb1318eedfb86",
      "75aee5bf8664480081d0e0778a2d6470",
      "6523047cf6e946b2b74864baf374d508"
     ]
    },
    "id": "_Jd6y8lDP9kY",
    "outputId": "5a8edcc0-b75c-4e3e-b50e-c7e6a5a72351"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b46ed1fb9c24fefa419ccb433abadab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/4.43k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f8522589dec4c9482404efe2c2bc25b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/21.8M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfeaaab2c1904277be9127ec8b18f87a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a90c1a09428458db59b0c27f858d1aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/12500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'content': 'You are an text to SQL query translator. Users will ask you questions in English and you will generate a SQL query based on the provided SCHEMA.\\nSCHEMA:\\nCREATE TABLE table_12962773_4 (no INTEGER, player VARCHAR)', 'role': 'system'}, {'content': 'What No is the player Zoran Erceg', 'role': 'user'}, {'content': 'SELECT MIN(no) FROM table_12962773_4 WHERE player = \"Zoran Erceg\"', 'role': 'assistant'}]\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# Convert dataset to OAI messages\n",
    "system_message = \"\"\"You are an text to SQL query translator. Users will ask you questions in English and you will generate a SQL query based on the provided SCHEMA.\n",
    "SCHEMA:\n",
    "{schema}\"\"\"\n",
    "\n",
    "def create_conversation(sample):\n",
    "  return {\n",
    "    \"messages\": [\n",
    "      {\"role\": \"system\", \"content\": system_message.format(schema=sample[\"context\"])},\n",
    "      {\"role\": \"user\", \"content\": sample[\"question\"]},\n",
    "      {\"role\": \"assistant\", \"content\": sample[\"answer\"]}\n",
    "    ]\n",
    "  }\n",
    "\n",
    "# Load dataset from the hub\n",
    "dataset = load_dataset(\"b-mc2/sql-create-context\", split=\"train\")\n",
    "dataset = dataset.shuffle().select(range(12500))\n",
    "\n",
    "# Convert dataset to OAI messages\n",
    "dataset = dataset.map(create_conversation, remove_columns=dataset.features,batched=False)\n",
    "# split dataset into 10,000 training samples and 2,500 test samples\n",
    "dataset = dataset.train_test_split(test_size=2500/12500)\n",
    "\n",
    "print(dataset[\"train\"][345][\"messages\"])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 133,
     "referenced_widgets": [
      "6d695dea42dd4a48b424e616c5ecdf8f",
      "e3fb4353b243443a8e71931c821be834",
      "7c1bcaeb31ee435fad9334278cf55d11",
      "cff0b8a169854a60b1805dd98373d529",
      "4b30a796bf584e69a3b2cbf923d157de",
      "3618f146843c4a9796014ec12e37cb9e",
      "732b83c78b464f5c853273acbfbf9d06",
      "d2d456c690e84a4f863e31607436147f",
      "be040f1c28564bf4b8dd369c012e376f",
      "016ff15cd42d44d4a5c59cdba3ed3098",
      "ae324e3a68d74d3488b59fcbf588f259",
      "31e23aadb45f476c8f50b91fb0a2db5a",
      "0d19627ff7f44cb49bbbdbce8f215600",
      "f408f30606444ca3b26969e52ebae5f0",
      "53f2a63470de49268d720f5705c73895",
      "88e8c44d8ff84c76a7225b233bf095c1",
      "25c18c66ad8b464c878980e03a656940",
      "91d70046a45e49778062c8fc7e828c35",
      "5bed68aff8c5433e8107d420290232d2",
      "3c2fb29e88bd4e2ba6423aaad342c352",
      "ebdc286ffdf14b37a8c75d86f422a6b6",
      "99ef2eef109a41ad9bad4fb22cf817fc"
     ]
    },
    "id": "o-B7kh9nQCVE",
    "outputId": "e9754f1a-d288-493d-ce33-23ee809b5a9e"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d695dea42dd4a48b424e616c5ecdf8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/10 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31e23aadb45f476c8f50b91fb0a2db5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data uploaded to:\n",
      "s3://sagemaker-ap-southeast-2-381491942612/datasets/text-to-sql/train_dataset.json\n",
      "https://s3.console.aws.amazon.com/s3/buckets/sagemaker-ap-southeast-2-381491942612/?region=ap-southeast-2&prefix=datasets/text-to-sql/\n"
     ]
    }
   ],
   "source": [
    "# save train_dataset to s3 using our SageMaker session\n",
    "training_input_path = f's3://{sess.default_bucket()}/datasets/text-to-sql'\n",
    "\n",
    "# save datasets to s3\n",
    "dataset[\"train\"].to_json(f\"{training_input_path}/train_dataset.json\", orient=\"records\")\n",
    "dataset[\"test\"].to_json(f\"{training_input_path}/test_dataset.json\", orient=\"records\")\n",
    "\n",
    "print(f\"Training data uploaded to:\")\n",
    "print(f\"{training_input_path}/train_dataset.json\")\n",
    "print(f\"https://s3.console.aws.amazon.com/s3/buckets/{sess.default_bucket()}/?region={sess.boto_region_name}&prefix={training_input_path.split('/', 3)[-1]}/\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z1FuRf6tQIlV"
   },
   "outputs": [],
   "source": [
    "# hyperparameters, which are passed into the training job\n",
    "hyperparameters = {\n",
    "  ### SCRIPT PARAMETERS ###\n",
    "  'dataset_path': '/opt/ml/input/data/training/train_dataset.json', # path where sagemaker will save training dataset\n",
    "  'model_id': \"codellama/CodeLlama-7b-hf\",           # or `mistralai/Mistral-7B-v0.1`\n",
    "  'max_seq_len': 3072,                               # max sequence length for model and packing of the dataset\n",
    "  'use_qlora': True,                                 # use QLoRA model\n",
    "  ### TRAINING PARAMETERS ###\n",
    "  'num_train_epochs': 3,                             # number of training epochs\n",
    "  'per_device_train_batch_size': 1,                  # batch size per device during training\n",
    "  'gradient_accumulation_steps': 4,                  # number of steps before performing a backward/update pass\n",
    "  'gradient_checkpointing': True,                    # use gradient checkpointing to save memory\n",
    "  'optim': \"adamw_torch_fused\",                      # use fused adamw optimizer\n",
    "  'logging_steps': 10,                               # log every 10 steps\n",
    "  'save_strategy': \"epoch\",                          # save checkpoint every epoch\n",
    "  'learning_rate': 2e-4,                             # learning rate, based on QLoRA paper\n",
    "  'bf16': False,                                      # use bfloat16 precision\n",
    "  'tf32': True,                                      # use tf32 precision\n",
    "  'max_grad_norm': 0.3,                              # max gradient norm based on QLoRA paper\n",
    "  'warmup_ratio': 0.03,                              # warmup ratio based on QLoRA paper\n",
    "  'lr_scheduler_type': \"constant\",                   # use constant learning rate scheduler\n",
    "  'report_to': \"tensorboard\",                        # report metrics to tensorboard\n",
    "  'output_dir': '/tmp/tun',                          # Temporary output directory for model checkpoints\n",
    "  'merge_adapters': True,                            # merge LoRA adapters into model for easier deployment\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6zBr3AC7QMpb"
   },
   "outputs": [],
   "source": [
    "from sagemaker.huggingface import HuggingFace\n",
    "\n",
    "# define Training Job Name\n",
    "job_name = f'codellama-7b-hf-text-to-sql-exp1'\n",
    "\n",
    "# create the Estimator\n",
    "huggingface_estimator = HuggingFace(\n",
    "    entry_point          = 'run_sft.py',    # train script\n",
    "    source_dir           = '/content/',#'https://github.com/philschmid/llm-sagemaker-sample/blob/main/scripts/trl',      # directory which includes all the files needed for training\n",
    "    instance_type        = 'ml.t3.medium',   # instances type used for the training job\n",
    "    instance_count       = 1,                 # the number of instances used for training\n",
    "    max_run              = 2*24*60*60,        # maximum runtime in seconds (days * hours * minutes * seconds)\n",
    "    base_job_name        = job_name,          # the name of the training job\n",
    "    role                 = role,              # Iam role used in training job to access AWS ressources, e.g. S3\n",
    "    volume_size          = 300,               # the size of the EBS volume in GB\n",
    "    transformers_version = '4.36',            # the transformers version used in the training job\n",
    "    pytorch_version      = '2.1',             # the pytorch_version version used in the training job\n",
    "    py_version           = 'py310',           # the python version used in the training job\n",
    "    hyperparameters      =  hyperparameters,  # the hyperparameters passed to the training job\n",
    "    disable_output_compression = True,        # not compress output to save training time and cost\n",
    "    environment          = {\n",
    "                            \"HUGGINGFACE_HUB_CACHE\": \"/tmp/.cache\", # set env variable to cache models in /tmp\n",
    "                             \"HF_TOKEN\": \"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\" # huggingface token to access gated models, e.g. llama 2\n",
    "                            },\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "37eXCIdgQ2lS"
   },
   "outputs": [],
   "source": [
    "# define a data input dictonary with our uploaded s3 uris\n",
    "data = {'training': training_input_path}\n",
    "\n",
    "# starting the train job with our uploaded datasets as input\n",
    "huggingface_estimator.fit(data, wait=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5YUBS7moRQKb",
    "outputId": "191a386d-0538-40e2-b3b9-6001241b3f4f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llm image uri: 763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/huggingface-pytorch-tgi-inference:2.1.1-tgi1.4.0-gpu-py310-cu121-ubuntu20.04\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.huggingface import get_huggingface_llm_image_uri\n",
    "\n",
    "# retrieve the llm image uri\n",
    "llm_image = get_huggingface_llm_image_uri(\n",
    "  \"huggingface\",\n",
    "  version=\"1.4.0\",\n",
    "  session=sess,\n",
    ")\n",
    "\n",
    "# print ecr image uri\n",
    "print(f\"llm image uri: {llm_image}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RaS_aDvAV1q6"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from sagemaker.huggingface import HuggingFaceModel\n",
    "\n",
    "# s3 path where the model will be uploaded\n",
    "# if you try to deploy the model to a different time add the s3 path here\n",
    "model_s3_path = huggingface_estimator.model_data[\"S3DataSource\"][\"S3Uri\"]\n",
    "\n",
    "# sagemaker config\n",
    "instance_type = \"ml.g5.2xlarge\"\n",
    "number_of_gpu = 1\n",
    "health_check_timeout = 300\n",
    "\n",
    "# Define Model and Endpoint configuration parameter\n",
    "config = {\n",
    "  'HF_MODEL_ID': \"/opt/ml/model\", # path to where sagemaker stores the model\n",
    "  'SM_NUM_GPUS': json.dumps(number_of_gpu), # Number of GPU used per replica\n",
    "  'MAX_INPUT_LENGTH': json.dumps(1024), # Max length of input text\n",
    "  'MAX_TOTAL_TOKENS': json.dumps(2048), # Max length of the generation (including input text)\n",
    "}\n",
    "\n",
    "# create HuggingFaceModel with the image uri\n",
    "llm_model = HuggingFaceModel(\n",
    "  role=role,\n",
    "  image_uri=llm_image,\n",
    "  model_data={'S3DataSource':{'S3Uri': model_s3_path,'S3DataType': 'S3Prefix','CompressionType': 'None'}},\n",
    "  env=config\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Na7ORSZPVtUn"
   },
   "outputs": [],
   "source": [
    "\n",
    "# Deploy model to an endpoint\n",
    "# https://sagemaker.readthedocs.io/en/stable/api/inference/model.html#sagemaker.model.Model.deploy\n",
    "llm = llm_model.deploy(\n",
    "  initial_instance_count=1,\n",
    "  instance_type=instance_type,\n",
    "  container_startup_health_check_timeout=health_check_timeout, # 10 minutes to give SageMaker the time to download the model\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HMElWI9CVY6V"
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "from sagemaker.s3 import S3Downloader\n",
    "\n",
    "# Load the tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"codellama/CodeLlama-7b-hf\")\n",
    "\n",
    "# Load the test dataset from s3\n",
    "S3Downloader.download(f\"{training_input_path}/test_dataset.json\", \".\")\n",
    "test_dataset = load_dataset(\"json\", data_files=\"test_dataset.json\",split=\"train\")\n",
    "random_sample = test_dataset[345]\n",
    "\n",
    "def request(sample):\n",
    "    prompt = tokenizer.apply_chat_template(sample, tokenize=False, add_generation_prompt=True)\n",
    "    outputs = llm.predict({\n",
    "      \"inputs\": prompt,\n",
    "      \"parameters\": {\n",
    "        \"max_new_tokens\": 512,\n",
    "        \"do_sample\": False,\n",
    "        \"return_full_text\": False,\n",
    "        \"stop\": [\"<|im_end|>\"],\n",
    "      }\n",
    "    })\n",
    "    return {\"role\": \"assistant\", \"content\": outputs[0][\"generated_text\"].strip()}\n",
    "\n",
    "print(random_sample[\"messages\"][1])\n",
    "request(random_sample[\"messages\"][:2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZwdaNVTthxIl"
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "from sagemaker.s3 import S3Downloader\n",
    "\n",
    "# Load the tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"codellama/CodeLlama-7b-hf\")\n",
    "\n",
    "# Load the test dataset from s3\n",
    "S3Downloader.download(f\"{training_input_path}/test_dataset.json\", \".\")\n",
    "test_dataset = load_dataset(\"json\", data_files=\"test_dataset.json\",split=\"train\")\n",
    "random_sample = test_dataset[345]\n",
    "\n",
    "def request(sample):\n",
    "    prompt = tokenizer.apply_chat_template(sample, tokenize=False, add_generation_prompt=True)\n",
    "    outputs = llm.predict({\n",
    "      \"inputs\": prompt,\n",
    "      \"parameters\": {\n",
    "        \"max_new_tokens\": 512,\n",
    "        \"do_sample\": False,\n",
    "        \"return_full_text\": False,\n",
    "        \"stop\": [\"<|im_end|>\"],\n",
    "      }\n",
    "    })\n",
    "    return {\"role\": \"assistant\", \"content\": outputs[0][\"generated_text\"].strip()}\n",
    "\n",
    "print(random_sample[\"messages\"][1])\n",
    "request(random_sample[\"messages\"][:2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ia3ZRbb-h_h6"
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def evaluate(sample):\n",
    "    predicted_answer = request(sample[\"messages\"][:2])\n",
    "    if predicted_answer[\"content\"] == sample[\"messages\"][2][\"content\"]:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "success_rate = []\n",
    "number_of_eval_samples = 1000\n",
    "# iterate over eval dataset and predict\n",
    "for s in tqdm(test_dataset.shuffle().select(range(number_of_eval_samples))):\n",
    "    success_rate.append(evaluate(s))\n",
    "\n",
    "# compute accuracy\n",
    "accuracy = sum(success_rate)/len(success_rate)\n",
    "\n",
    "print(f\"Accuracy: {accuracy*100:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "thGQfIyZiAcH"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'llm' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m llm\u001b[38;5;241m.\u001b[39mdelete_model()\n\u001b[0;32m      2\u001b[0m llm\u001b[38;5;241m.\u001b[39mdelete_endpoint()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'llm' is not defined"
     ]
    }
   ],
   "source": [
    "llm.delete_model()\n",
    "llm.delete_endpoint()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "016ff15cd42d44d4a5c59cdba3ed3098": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "019bbb311b6b4f5fa5d50dc18b19e0bf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "028ce9a6bf8a4633b04f35eac230fd40": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_40503eed415d4b00a8bf4a7fba52d43b",
      "placeholder": "​",
      "style": "IPY_MODEL_86769be2ea6544488b576b2198b49b34",
      "value": "Generating train split: "
     }
    },
    "035962d8eacb4a7fa8bffdc85b511a33": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "0765ed8920494d148eaaf164a9c2636b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "0d19627ff7f44cb49bbbdbce8f215600": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_25c18c66ad8b464c878980e03a656940",
      "placeholder": "​",
      "style": "IPY_MODEL_91d70046a45e49778062c8fc7e828c35",
      "value": "Creating json from Arrow format: 100%"
     }
    },
    "17d04efd0ebd482698e4ee35ce37b26b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "25c18c66ad8b464c878980e03a656940": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "31e23aadb45f476c8f50b91fb0a2db5a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0d19627ff7f44cb49bbbdbce8f215600",
       "IPY_MODEL_f408f30606444ca3b26969e52ebae5f0",
       "IPY_MODEL_53f2a63470de49268d720f5705c73895"
      ],
      "layout": "IPY_MODEL_88e8c44d8ff84c76a7225b233bf095c1"
     }
    },
    "3618f146843c4a9796014ec12e37cb9e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3c2fb29e88bd4e2ba6423aaad342c352": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "40503eed415d4b00a8bf4a7fba52d43b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4119bcd74693409ba223a0b14a1bfcc1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4b30a796bf584e69a3b2cbf923d157de": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4ec195abd6f74d20bb14eb29280d01ff": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4febf57d3514478dbf0a5294f633e442": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "53f2a63470de49268d720f5705c73895": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ebdc286ffdf14b37a8c75d86f422a6b6",
      "placeholder": "​",
      "style": "IPY_MODEL_99ef2eef109a41ad9bad4fb22cf817fc",
      "value": " 3/3 [00:00&lt;00:00, 27.46ba/s]"
     }
    },
    "5bed68aff8c5433e8107d420290232d2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6464e81f20404c91b5d263d0b09d9d00": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6523047cf6e946b2b74864baf374d508": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6741c2fc6cd7458aa7da7e96d4f46516": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6d695dea42dd4a48b424e616c5ecdf8f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e3fb4353b243443a8e71931c821be834",
       "IPY_MODEL_7c1bcaeb31ee435fad9334278cf55d11",
       "IPY_MODEL_cff0b8a169854a60b1805dd98373d529"
      ],
      "layout": "IPY_MODEL_4b30a796bf584e69a3b2cbf923d157de"
     }
    },
    "6e2ad038f7c54a8995b071d203ae33a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6e7233b59b0c4a46b99deeea81380319": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "724ccb94d17840a49441b02be4e6a374": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "732b83c78b464f5c853273acbfbf9d06": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7453c7e0b8c046a699283cab05cfa934": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c99e46458fa14bbea865e7cc50455d8c",
      "max": 12500,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e9d262277bd4409d8d4cb1318eedfb86",
      "value": 12500
     }
    },
    "75aee5bf8664480081d0e0778a2d6470": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7b46ed1fb9c24fefa419ccb433abadab": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c56f34bd458441299bf81140b4cfb972",
       "IPY_MODEL_a3c66a59194b4b2899b98efe4d4eb2f8",
       "IPY_MODEL_ba7e2206c665461496e40357e41f41d1"
      ],
      "layout": "IPY_MODEL_4febf57d3514478dbf0a5294f633e442"
     }
    },
    "7ba369d41893482fac304c9ac3775d8d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fdd46bf54caa463cb296a39ae0d9b663",
      "placeholder": "​",
      "style": "IPY_MODEL_6741c2fc6cd7458aa7da7e96d4f46516",
      "value": "Downloading data: 100%"
     }
    },
    "7c1bcaeb31ee435fad9334278cf55d11": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d2d456c690e84a4f863e31607436147f",
      "max": 10,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_be040f1c28564bf4b8dd369c012e376f",
      "value": 10
     }
    },
    "7f8522589dec4c9482404efe2c2bc25b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7ba369d41893482fac304c9ac3775d8d",
       "IPY_MODEL_fd313d073c8847b9bb747a9f27ca9cc0",
       "IPY_MODEL_a2d212ca60fb478e837912c1d8e8635b"
      ],
      "layout": "IPY_MODEL_6e7233b59b0c4a46b99deeea81380319"
     }
    },
    "86769be2ea6544488b576b2198b49b34": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "880614c09f7e429e9ce2257d2e3ab068": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88e8c44d8ff84c76a7225b233bf095c1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "91d70046a45e49778062c8fc7e828c35": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "99ef2eef109a41ad9bad4fb22cf817fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9a90c1a09428458db59b0c27f858d1aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_a2e1ea30d77e40fbaaac91c9fcee9671",
       "IPY_MODEL_7453c7e0b8c046a699283cab05cfa934",
       "IPY_MODEL_e066643b5c40437baf6bdfe00960f210"
      ],
      "layout": "IPY_MODEL_6464e81f20404c91b5d263d0b09d9d00"
     }
    },
    "9b842af5381f47a897498d4304e8e4f3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "20px"
     }
    },
    "a13603c15ce5469c90b7c65ff9105b81": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a20c87d57f1b4c7288a8dfea6294f857": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9b842af5381f47a897498d4304e8e4f3",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_035962d8eacb4a7fa8bffdc85b511a33",
      "value": 1
     }
    },
    "a2210d478b1c4782b11745884e9240d6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a2d212ca60fb478e837912c1d8e8635b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d8e9a52f3b784e30a6e2a9d0a1208caf",
      "placeholder": "​",
      "style": "IPY_MODEL_a2210d478b1c4782b11745884e9240d6",
      "value": " 21.8M/21.8M [00:00&lt;00:00, 33.1MB/s]"
     }
    },
    "a2e1ea30d77e40fbaaac91c9fcee9671": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b433fce091d143f782f9a21850bc9427",
      "placeholder": "​",
      "style": "IPY_MODEL_6e2ad038f7c54a8995b071d203ae33a6",
      "value": "Map: 100%"
     }
    },
    "a3c66a59194b4b2899b98efe4d4eb2f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a13603c15ce5469c90b7c65ff9105b81",
      "max": 4432,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0765ed8920494d148eaaf164a9c2636b",
      "value": 4432
     }
    },
    "ac9a8ac5f9f642e883de07c3c4696e36": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ae324e3a68d74d3488b59fcbf588f259": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b433fce091d143f782f9a21850bc9427": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ba7e2206c665461496e40357e41f41d1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_17d04efd0ebd482698e4ee35ce37b26b",
      "placeholder": "​",
      "style": "IPY_MODEL_ac9a8ac5f9f642e883de07c3c4696e36",
      "value": " 4.43k/4.43k [00:00&lt;00:00, 148kB/s]"
     }
    },
    "be040f1c28564bf4b8dd369c012e376f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "bfeaaab2c1904277be9127ec8b18f87a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_028ce9a6bf8a4633b04f35eac230fd40",
       "IPY_MODEL_a20c87d57f1b4c7288a8dfea6294f857",
       "IPY_MODEL_e751748cd25b407b9a4dded11b168554"
      ],
      "layout": "IPY_MODEL_4ec195abd6f74d20bb14eb29280d01ff"
     }
    },
    "c56f34bd458441299bf81140b4cfb972": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_019bbb311b6b4f5fa5d50dc18b19e0bf",
      "placeholder": "​",
      "style": "IPY_MODEL_e4cd0e357d0d4dcc9d58fa3db0dff299",
      "value": "Downloading readme: 100%"
     }
    },
    "c99e46458fa14bbea865e7cc50455d8c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cff0b8a169854a60b1805dd98373d529": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_016ff15cd42d44d4a5c59cdba3ed3098",
      "placeholder": "​",
      "style": "IPY_MODEL_ae324e3a68d74d3488b59fcbf588f259",
      "value": " 10/10 [00:00&lt;00:00, 35.23ba/s]"
     }
    },
    "d2d456c690e84a4f863e31607436147f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d46dc1ef87ca4297b9c47c7eacdfb0d4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "d8e9a52f3b784e30a6e2a9d0a1208caf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e066643b5c40437baf6bdfe00960f210": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_75aee5bf8664480081d0e0778a2d6470",
      "placeholder": "​",
      "style": "IPY_MODEL_6523047cf6e946b2b74864baf374d508",
      "value": " 12500/12500 [00:01&lt;00:00, 7468.39 examples/s]"
     }
    },
    "e3fb4353b243443a8e71931c821be834": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3618f146843c4a9796014ec12e37cb9e",
      "placeholder": "​",
      "style": "IPY_MODEL_732b83c78b464f5c853273acbfbf9d06",
      "value": "Creating json from Arrow format: 100%"
     }
    },
    "e4cd0e357d0d4dcc9d58fa3db0dff299": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e751748cd25b407b9a4dded11b168554": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_880614c09f7e429e9ce2257d2e3ab068",
      "placeholder": "​",
      "style": "IPY_MODEL_4119bcd74693409ba223a0b14a1bfcc1",
      "value": " 78577/0 [00:00&lt;00:00, 116378.97 examples/s]"
     }
    },
    "e9d262277bd4409d8d4cb1318eedfb86": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ebdc286ffdf14b37a8c75d86f422a6b6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f408f30606444ca3b26969e52ebae5f0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5bed68aff8c5433e8107d420290232d2",
      "max": 3,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3c2fb29e88bd4e2ba6423aaad342c352",
      "value": 3
     }
    },
    "fd313d073c8847b9bb747a9f27ca9cc0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_724ccb94d17840a49441b02be4e6a374",
      "max": 21752418,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_d46dc1ef87ca4297b9c47c7eacdfb0d4",
      "value": 21752418
     }
    },
    "fdd46bf54caa463cb296a39ae0d9b663": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}