2b - En-Hi_Sequence to Sequence Learning with Neural Networks-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T05:02:28.651625Z",
     "iopub.status.busy": "2022-02-14T05:02:28.651420Z",
     "iopub.status.idle": "2022-02-14T05:02:28.655075Z",
     "shell.execute_reply": "2022-02-14T05:02:28.654584Z",
     "shell.execute_reply.started": "2022-02-14T05:02:28.651593Z"
    },
    "id": "bJEMnvNc2OCD"
   },
   "source": [
    "<font color = 'red'  size = 4>**Unlike the original repo , I have used reversed tokens for source**</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jMIVwQyitAXd"
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9589,
     "status": "ok",
     "timestamp": 1644819607633,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "qzjAVhD44yZC",
    "outputId": "8070452e-eb54-4f08-ee3d-4502b4354e81"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pickle5\n",
      "  Downloading pickle5-0.0.12-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (256 kB)\n",
      "\u001b[K     |████████████████████████████████| 256 kB 8.0 MB/s eta 0:00:01\n",
      "\u001b[?25hInstalling collected packages: pickle5\n",
      "Successfully installed pickle5-0.0.12\n"
     ]
    }
   ],
   "source": [
    "# required for colab only\n",
    "# commen this for local\n",
    "#!pip3 install pickle5\n",
    "#import pickle5 as pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:05.490885Z",
     "iopub.status.busy": "2022-02-14T07:45:05.490743Z",
     "iopub.status.idle": "2022-02-14T07:45:05.493469Z",
     "shell.execute_reply": "2022-02-14T07:45:05.492967Z",
     "shell.execute_reply.started": "2022-02-14T07:45:05.490872Z"
    },
    "id": "xnCt_pBz42zN",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# for local laptop uncomment this abd comment above cell\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GNnMugG1WKTm"
   },
   "outputs": [],
   "source": [
    "#!pip install -U spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Hp1WKED5WLNu"
   },
   "outputs": [],
   "source": [
    "#!pip install -U swifter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-03T07:18:13.481858Z",
     "iopub.status.busy": "2022-04-03T07:18:13.481680Z",
     "iopub.status.idle": "2022-04-03T07:18:14.284326Z",
     "shell.execute_reply": "2022-04-03T07:18:14.283948Z",
     "shell.execute_reply.started": "2022-04-03T07:18:13.481804Z"
    },
    "executionInfo": {
     "elapsed": 7585,
     "status": "ok",
     "timestamp": 1644819666705,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "JiUqMdpCUn24",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.nn.utils import clip_grad_norm_, clip_grad_value_\n",
    "\n",
    "import torchtext \n",
    "from torchtext.vocab import vocab\n",
    "#from torchtext.legacy.data import Field, BucketIterator\n",
    "\n",
    "import spacy\n",
    "import numpy as np\n",
    "from collections import Counter, OrderedDict\n",
    "\n",
    "import random\n",
    "import math\n",
    "import time\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "#import swifter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:43:40.953887Z",
     "iopub.status.busy": "2022-02-14T07:43:40.953439Z",
     "iopub.status.idle": "2022-02-14T07:43:40.955626Z",
     "shell.execute_reply": "2022-02-14T07:43:40.955222Z",
     "shell.execute_reply.started": "2022-02-14T07:43:40.953854Z"
    },
    "executionInfo": {
     "elapsed": 19589,
     "status": "ok",
     "timestamp": 1644819691919,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "9uY_ZBxdtH_k",
    "outputId": "c145ef02-3003-4edd-88b5-5c3763ede25b",
    "tags": []
   },
   "outputs": [],
   "source": [
    "#from google.colab import drive\n",
    "#drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-03T07:18:16.490959Z",
     "iopub.status.busy": "2022-04-03T07:18:16.490819Z",
     "iopub.status.idle": "2022-04-03T07:18:16.493311Z",
     "shell.execute_reply": "2022-04-03T07:18:16.492959Z",
     "shell.execute_reply.started": "2022-04-03T07:18:16.490945Z"
    },
    "executionInfo": {
     "elapsed": 324,
     "status": "ok",
     "timestamp": 1644819696523,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "LLLu5aF_VkRw",
    "tags": []
   },
   "outputs": [],
   "source": [
    "folder = Path('/home/harpreet/Insync/google_drive_harpreet/Research/NLP/pytorch-seq2seq')\n",
    "#folder = Path('/content/drive/MyDrive/Research/NLP/pytorch-seq2seq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-04-03T07:18:16.860914Z",
     "iopub.status.busy": "2022-04-03T07:18:16.860711Z",
     "iopub.status.idle": "2022-04-03T07:18:16.870241Z",
     "shell.execute_reply": "2022-04-03T07:18:16.869807Z",
     "shell.execute_reply.started": "2022-04-03T07:18:16.860882Z"
    },
    "executionInfo": {
     "elapsed": 208,
     "status": "ok",
     "timestamp": 1644819697740,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "PY9-guO2Un25",
    "outputId": "3c93c789-4eae-4578-f783-a0e44a4b48e2",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0.11.0', '1.10.0', True, '3.2.4')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torchtext.__version__, torch.__version__, torch.cuda.is_available(), spacy.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wRVU8aBPUn26"
   },
   "source": [
    "# Set Seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-03T07:18:20.530681Z",
     "iopub.status.busy": "2022-04-03T07:18:20.530552Z",
     "iopub.status.idle": "2022-04-03T07:18:20.534113Z",
     "shell.execute_reply": "2022-04-03T07:18:20.533727Z",
     "shell.execute_reply.started": "2022-04-03T07:18:20.530667Z"
    },
    "executionInfo": {
     "elapsed": 180,
     "status": "ok",
     "timestamp": 1644819702627,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "vzCQltiuUn26",
    "tags": []
   },
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EQFjo0_dUL2I"
   },
   "source": [
    "# Load Tokenized Data\n",
    "Next, we download and load the train, validation and test data. \n",
    "\n",
    "The dataset we'll be using is the [IIT Bombay English-Hindi Corpus](https://www.cfilt.iitb.ac.in/iitb_parallel/). \n",
    "Anoop Kunchukuttan, Pratik Mehta, Pushpak Bhattacharyya. The IIT Bombay English-Hindi Parallel Corpus. Language Resources and Evaluation Conference. 2018.\n",
    "\n",
    "The datset can be  downloaded from huggingface library as well. \n",
    "We have downloaded and tokenized the data in Tutorial 1. We will now load the tokenized data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2022-02-14T07:44:15.702006Z",
     "iopub.status.busy": "2022-02-14T07:44:15.701865Z",
     "iopub.status.idle": "2022-02-14T07:44:15.704867Z",
     "shell.execute_reply": "2022-02-14T07:44:15.704583Z",
     "shell.execute_reply.started": "2022-02-14T07:44:15.701992Z"
    },
    "executionInfo": {
     "elapsed": 23056,
     "status": "ok",
     "timestamp": 1644819771777,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "yZFv_IGh2_X7",
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nwith open(folder/\\'df_train_hi_en\\', \"rb\") as fh:\\n  df_train = pickle.load(fh)\\nwith open(folder/\\'df_test_hi_en\\', \"rb\") as fh:\\n  df_test = pickle.load(fh)\\nwith open(folder/\\'df_valid_hi_en\\', \"rb\") as fh:\\n  df_valid = pickle.load(fh)  \\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Required for colab only\n",
    "'''\n",
    "with open(folder/'df_train_hi_en', \"rb\") as fh:\n",
    "  df_train = pickle.load(fh)\n",
    "with open(folder/'df_test_hi_en', \"rb\") as fh:\n",
    "  df_test = pickle.load(fh)\n",
    "with open(folder/'df_valid_hi_en', \"rb\") as fh:\n",
    "  df_valid = pickle.load(fh)  \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-03T07:19:28.170293Z",
     "iopub.status.busy": "2022-04-03T07:19:28.170151Z",
     "iopub.status.idle": "2022-04-03T07:19:39.547737Z",
     "shell.execute_reply": "2022-04-03T07:19:39.547412Z",
     "shell.execute_reply.started": "2022-04-03T07:19:28.170279Z"
    },
    "id": "Uh71ereCl23K",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# required for local (python 3.8)\n",
    "df_train = pd.read_pickle(folder/'df_train_hi_en')\n",
    "df_test = pd.read_pickle(folder/'df_test_hi_en')\n",
    "df_valid = pd.read_pickle(folder/'df_valid_hi_en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "execution": {
     "iopub.execute_input": "2022-04-03T07:19:39.548435Z",
     "iopub.status.busy": "2022-04-03T07:19:39.548333Z",
     "iopub.status.idle": "2022-04-03T07:19:39.557327Z",
     "shell.execute_reply": "2022-04-03T07:19:39.557020Z",
     "shell.execute_reply.started": "2022-04-03T07:19:39.548422Z"
    },
    "executionInfo": {
     "elapsed": 208,
     "status": "ok",
     "timestamp": 1644819778943,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "n4WaFnDVl23K",
    "outputId": "49a23515-c33c-4962-884c-65a598b50e71",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_tokens_reversed</th>\n",
       "      <th>target_tokens</th>\n",
       "      <th>source_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[दें, लाभ, का, व्यायाम, पहुंचनीयता, को, अनुप्र...</td>\n",
       "      <td>[Give, your, application, an, accessibility, w...</td>\n",
       "      <td>[अपने, अनुप्रयोग, को, पहुंचनीयता, व्यायाम, का,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[अन्वेषक, पहुंचनीयता, एक्सेर्साइसर]</td>\n",
       "      <td>[Accerciser, Accessibility, Explorer]</td>\n",
       "      <td>[एक्सेर्साइसर, पहुंचनीयता, अन्वेषक]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[खाका, प्लग-इन, डिफोल्ट, लिए, के, पटल, निचले]</td>\n",
       "      <td>[The, default, plugin, layout, for, the, botto...</td>\n",
       "      <td>[निचले, पटल, के, लिए, डिफोल्ट, प्लग-इन, खाका]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[खाका, प्लग-इन, डिफोल्ट, लिए, के, पटल, ऊपरी]</td>\n",
       "      <td>[The, default, plugin, layout, for, the, top, ...</td>\n",
       "      <td>[ऊपरी, पटल, के, लिए, डिफोल्ट, प्लग-इन, खाका]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[है, गया, किया, निष्क्रिय, से, रूप, डिफोल्ट, ज...</td>\n",
       "      <td>[A, list, of, plugins, that, are, disabled, by...</td>\n",
       "      <td>[उन, प्लग-इनों, की, सूची, जिन्हें, डिफोल्ट, रू...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              source_tokens_reversed  \\\n",
       "0  [दें, लाभ, का, व्यायाम, पहुंचनीयता, को, अनुप्र...   \n",
       "1                [अन्वेषक, पहुंचनीयता, एक्सेर्साइसर]   \n",
       "2      [खाका, प्लग-इन, डिफोल्ट, लिए, के, पटल, निचले]   \n",
       "3       [खाका, प्लग-इन, डिफोल्ट, लिए, के, पटल, ऊपरी]   \n",
       "4  [है, गया, किया, निष्क्रिय, से, रूप, डिफोल्ट, ज...   \n",
       "\n",
       "                                       target_tokens  \\\n",
       "0  [Give, your, application, an, accessibility, w...   \n",
       "1              [Accerciser, Accessibility, Explorer]   \n",
       "2  [The, default, plugin, layout, for, the, botto...   \n",
       "3  [The, default, plugin, layout, for, the, top, ...   \n",
       "4  [A, list, of, plugins, that, are, disabled, by...   \n",
       "\n",
       "                                       source_tokens  \n",
       "0  [अपने, अनुप्रयोग, को, पहुंचनीयता, व्यायाम, का,...  \n",
       "1                [एक्सेर्साइसर, पहुंचनीयता, अन्वेषक]  \n",
       "2      [निचले, पटल, के, लिए, डिफोल्ट, प्लग-इन, खाका]  \n",
       "3       [ऊपरी, पटल, के, लिए, डिफोल्ट, प्लग-इन, खाका]  \n",
       "4  [उन, प्लग-इनों, की, सूची, जिन्हें, डिफोल्ट, रू...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1BEf_jK2Tz6E"
   },
   "source": [
    "# Load Vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:13.073242Z",
     "iopub.status.busy": "2022-02-14T07:45:13.073096Z",
     "iopub.status.idle": "2022-02-14T07:45:13.135072Z",
     "shell.execute_reply": "2022-02-14T07:45:13.134733Z",
     "shell.execute_reply.started": "2022-02-14T07:45:13.073228Z"
    },
    "executionInfo": {
     "elapsed": 1161,
     "status": "ok",
     "timestamp": 1644819781707,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "cvsVAiyHl23M",
    "tags": []
   },
   "outputs": [],
   "source": [
    "source_vocab = pickle.load(open(folder/'source_vocab_hi_en.pkl','rb'))\n",
    "target_vocab = pickle.load(open(folder/'target_vocab_hi_en.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:14.031078Z",
     "iopub.status.busy": "2022-02-14T07:45:14.030939Z",
     "iopub.status.idle": "2022-02-14T07:45:14.034097Z",
     "shell.execute_reply": "2022-02-14T07:45:14.033754Z",
     "shell.execute_reply.started": "2022-02-14T07:45:14.031065Z"
    },
    "executionInfo": {
     "elapsed": 211,
     "status": "ok",
     "timestamp": 1644819787547,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "SprHodI2l23M",
    "outputId": "4c41a1ee-15c8-455e-c6b4-f754a949bd9e",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_vocab['from']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:14.585923Z",
     "iopub.status.busy": "2022-02-14T07:45:14.585772Z",
     "iopub.status.idle": "2022-02-14T07:45:14.588900Z",
     "shell.execute_reply": "2022-02-14T07:45:14.588648Z",
     "shell.execute_reply.started": "2022-02-14T07:45:14.585909Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1644819788471,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "yJjpoffol23M",
    "outputId": "820fc3b3-831c-47c2-c577-fd3b44ae0713",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6115, 6537)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(source_vocab), len(target_vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "feccYoeWtAXk"
   },
   "source": [
    "# Create Dataset and Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:15.920782Z",
     "iopub.status.busy": "2022-02-14T07:45:15.920643Z",
     "iopub.status.idle": "2022-02-14T07:45:15.923921Z",
     "shell.execute_reply": "2022-02-14T07:45:15.923576Z",
     "shell.execute_reply.started": "2022-02-14T07:45:15.920769Z"
    },
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1644819790198,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "fCMXepKUtAXk",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class EngHindi(Dataset):\n",
    "    \n",
    "    '''\n",
    "    Takes input as (X1, X2)\n",
    "    X1 : pandas series for  source language\n",
    "    X2 : pndas series for target language\n",
    "    '''\n",
    "    def __init__(self, X1, X2):\n",
    "        self.X1 = X1\n",
    "        self.X2 = X2\n",
    "        \n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.X1)\n",
    "    \n",
    "    def __getitem__(self, indices):\n",
    "        source_examples = self.X1.iloc[indices]  \n",
    "        target_examples = self.X2.iloc[indices]\n",
    "        return source_examples, target_examples    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:16.625947Z",
     "iopub.status.busy": "2022-02-14T07:45:16.625794Z",
     "iopub.status.idle": "2022-02-14T07:45:16.628858Z",
     "shell.execute_reply": "2022-02-14T07:45:16.628588Z",
     "shell.execute_reply.started": "2022-02-14T07:45:16.625933Z"
    },
    "executionInfo": {
     "elapsed": 174,
     "status": "ok",
     "timestamp": 1644819793874,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "-Q32VBamtAXl",
    "tags": []
   },
   "outputs": [],
   "source": [
    "trainset = EngHindi(df_train['source_tokens'], df_train['target_tokens'])\n",
    "testset =  EngHindi(df_test['source_tokens'], df_test['target_tokens'])\n",
    "validset = EngHindi(df_valid['source_tokens'], df_valid['target_tokens'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:17.060595Z",
     "iopub.status.busy": "2022-02-14T07:45:17.060448Z",
     "iopub.status.idle": "2022-02-14T07:45:17.063876Z",
     "shell.execute_reply": "2022-02-14T07:45:17.063550Z",
     "shell.execute_reply.started": "2022-02-14T07:45:17.060581Z"
    },
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1644819794799,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "2md5tDSutAXl",
    "outputId": "31329144-c410-4e1c-a379-22c5ab302d1a",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['दें', 'लाभ', 'का', 'व्यायाम', 'पहुंचनीयता', 'को', 'अनुप्रयोग', 'अपने'],\n",
       " ['Give', 'your', 'application', 'an', 'accessibility', 'workout'])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset.__getitem__(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:17.645538Z",
     "iopub.status.busy": "2022-02-14T07:45:17.645420Z",
     "iopub.status.idle": "2022-02-14T07:45:17.648617Z",
     "shell.execute_reply": "2022-02-14T07:45:17.648236Z",
     "shell.execute_reply.started": "2022-02-14T07:45:17.645525Z"
    },
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1644819795017,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "L18xzZrltAXl",
    "outputId": "7299f992-8806-4107-b65e-3b171af2c00c",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1659083, 2507, 520)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trainset), len(testset), len(validset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:18.125158Z",
     "iopub.status.busy": "2022-02-14T07:45:18.124964Z",
     "iopub.status.idle": "2022-02-14T07:45:18.128231Z",
     "shell.execute_reply": "2022-02-14T07:45:18.127877Z",
     "shell.execute_reply.started": "2022-02-14T07:45:18.125144Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1644819795736,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "TehVY0IoQUvb",
    "outputId": "018bcfe5-388c-457b-e27b-2d633470a873",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33181.66"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trainset)*0.02"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:18.799884Z",
     "iopub.status.busy": "2022-02-14T07:45:18.799761Z",
     "iopub.status.idle": "2022-02-14T07:45:18.813086Z",
     "shell.execute_reply": "2022-02-14T07:45:18.812633Z",
     "shell.execute_reply.started": "2022-02-14T07:45:18.799871Z"
    },
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1644819797021,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "wDzZ5HlFQOYA",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# get subset of data\n",
    "# We will be using only 100 images for nboth train and validation datasets\n",
    "train_sample_size = int(len(trainset)*0.02)\n",
    "\n",
    "# Getting n random indices\n",
    "train_subset_indices = random.sample(range(0, len(trainset)), train_sample_size)\n",
    "\n",
    "# Getting subset of dataset\n",
    "train_subset = torch.utils.data.Subset(trainset, train_subset_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:19.205347Z",
     "iopub.status.busy": "2022-02-14T07:45:19.205204Z",
     "iopub.status.idle": "2022-02-14T07:45:19.208459Z",
     "shell.execute_reply": "2022-02-14T07:45:19.208086Z",
     "shell.execute_reply.started": "2022-02-14T07:45:19.205333Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1644819798192,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "D5ALqVW9QsUz",
    "outputId": "557ae2e7-404a-4537-fc6d-2cbf90c3d174",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(['।', 'होगा', 'सुनिश्चित', 'भविष्य', 'का', 'राष्ट्र', 'हमारे', 'और', 'बढ़ेगा', 'आधार', 'का', 'पीरामिड', 'नवाचार', 'हमारे', 'से', 'बोने', 'बीज', 'के', 'शक्ति', 'की', 'नवाचारों', 'और', 'विचारों', 'नए', 'में', 'बच्चों', 'स्कूली'], ['Seeding', 'the', 'power', 'of', 'ideas', 'and', 'innovation', 'in', 'schoolchildren', 'will', 'broaden', 'the', 'base', 'of', 'our', 'innovation', 'pyramid', 'and', 'secure', 'the', 'future', 'of', 'our', 'nation', '.'])\n"
     ]
    }
   ],
   "source": [
    "print(train_subset.__getitem__(11))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:19.987277Z",
     "iopub.status.busy": "2022-02-14T07:45:19.987150Z",
     "iopub.status.idle": "2022-02-14T07:45:19.989872Z",
     "shell.execute_reply": "2022-02-14T07:45:19.989433Z",
     "shell.execute_reply.started": "2022-02-14T07:45:19.987262Z"
    },
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1644819799431,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "bPg8WslltAXl",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# transform text to indexes and append eos and bos\n",
    "# finally convert to tensors\n",
    "def text_transform(my_vocab, text):\n",
    "    text_num = [my_vocab['<BOS>']] + [my_vocab[word] for word in text] + [my_vocab['<EOS>']]\n",
    "    return torch.tensor(text_num)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:20.675563Z",
     "iopub.status.busy": "2022-02-14T07:45:20.675434Z",
     "iopub.status.idle": "2022-02-14T07:45:20.678119Z",
     "shell.execute_reply": "2022-02-14T07:45:20.677767Z",
     "shell.execute_reply.started": "2022-02-14T07:45:20.675549Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1644819800754,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "bpQKiRHLtAXl",
    "outputId": "6c8aa468-2161-46cd-fda5-744c0599289c",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Bad', 'current', 'tag', 'value', '.']\n"
     ]
    }
   ],
   "source": [
    "text = train_subset.__getitem__(13)[1]\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:21.095520Z",
     "iopub.status.busy": "2022-02-14T07:45:21.095381Z",
     "iopub.status.idle": "2022-02-14T07:45:21.099201Z",
     "shell.execute_reply": "2022-02-14T07:45:21.098826Z",
     "shell.execute_reply.started": "2022-02-14T07:45:21.095506Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1644819801675,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "-QZp7ZlAl23N",
    "outputId": "d29daef0-6f1d-4b33-be36-8527a6884736",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([   1, 1032,   37,  672,  104,   33,    2])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_transform(target_vocab, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:21.739861Z",
     "iopub.status.busy": "2022-02-14T07:45:21.739738Z",
     "iopub.status.idle": "2022-02-14T07:45:21.743219Z",
     "shell.execute_reply": "2022-02-14T07:45:21.742893Z",
     "shell.execute_reply.started": "2022-02-14T07:45:21.739847Z"
    },
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1644819802264,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "CgKupdZ1Un2-",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def collate_batch(batch):\n",
    "    source_list, target_list = [], []\n",
    "    for source, target in batch:\n",
    "        source_tensor = text_transform(source_vocab, source)\n",
    "        target_tensor = text_transform(target_vocab, target)\n",
    "        source_list.append(source_tensor)\n",
    "        target_list.append(target_tensor)\n",
    "        \n",
    "    source_pad = pad_sequence(source_list, batch_first=False, padding_value= source_vocab['<PAD>'])\n",
    "    target_pad = pad_sequence(target_list, batch_first=False, padding_value= target_vocab['<PAD>'])\n",
    "    \n",
    "    return source_pad, target_pad     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:45:22.505120Z",
     "iopub.status.busy": "2022-02-14T07:45:22.504910Z",
     "iopub.status.idle": "2022-02-14T07:45:22.507920Z",
     "shell.execute_reply": "2022-02-14T07:45:22.507648Z",
     "shell.execute_reply.started": "2022-02-14T07:45:22.505066Z"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1644819803722,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "axyZQ-4z1B-z",
    "tags": []
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "\n",
    "train_loader = DataLoader(train_subset, batch_size=BATCH_SIZE, shuffle = True, collate_fn = collate_batch )\n",
    "valid_loader = DataLoader(validset, batch_size=BATCH_SIZE, shuffle = False, collate_fn = collate_batch )\n",
    "test_loader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle = False, collate_fn = collate_batch )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MjTMLaX0Un22"
   },
   "source": [
    "# 2a - Sequence to Sequence Learning with Neural Networks\n",
    "\n",
    "## Introduction\n",
    "\n",
    "Let's remind ourselves of the general encoder-decoder model.\n",
    "\n",
    "![](assets/seq2seq1.png)\n",
    "\n",
    "We use our encoder (green) over the embedded source sequence (yellow) to create a context vector (red). We then use that context vector with the decoder (blue) and a linear layer (purple) to generate the target sentence.\n",
    "\n",
    "In the previous model, we used an multi-layered LSTM as the encoder and decoder.\n",
    "\n",
    "![](assets/seq2seq4.png)\n",
    "\n",
    "One downside of the previous model is that the decoder is trying to cram lots of information into the hidden states. Whilst decoding, the hidden state will need to contain information about the whole of the source sequence, as well as all of the tokens have been decoded so far. By alleviating some of this information compression, we can create a better model!\n",
    "\n",
    "We'll also be using a GRU (Gated Recurrent Unit) instead of an LSTM (Long Short-Term Memory). Why? Mainly because that's what they did in the paper (this paper also introduced GRUs) and also because we used LSTMs last time. To understand how GRUs (and LSTMs) differ from standard RNNS, check out [this](https://colah.github.io/posts/2015-08-Understanding-LSTMs/) link. Is a GRU better than an LSTM? [Research](https://arxiv.org/abs/1412.3555) has shown they're pretty much the same, and both are better than standard RNNs. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Mhyhh7uCUn2_"
   },
   "source": [
    "## Building the Seq2Seq Model\n",
    "\n",
    "### Encoder\n",
    "\n",
    "The encoder is similar to the previous one, with the multi-layer LSTM swapped for a single-layer GRU. We also don't pass the dropout as an argument to the GRU as that dropout is used between each layer of a multi-layered RNN. As we only have a single layer, PyTorch will display a warning if we try and use pass a dropout value to it.\n",
    "\n",
    "Another thing to note about the GRU is that it only requires and returns a hidden state, there is no cell state like in the LSTM.\n",
    "\n",
    "$$\\begin{align*}\n",
    "h_t &= \\text{GRU}(e(x_t), h_{t-1})\\\\\n",
    "(h_t, c_t) &= \\text{LSTM}(e(x_t), h_{t-1}, c_{t-1})\\\\\n",
    "h_t &= \\text{RNN}(e(x_t), h_{t-1})\n",
    "\\end{align*}$$\n",
    "\n",
    "From the equations above, it looks like the RNN and the GRU are identical. Inside the GRU, however, is a number of *gating mechanisms* that control the information flow in to and out of the hidden state (similar to an LSTM). Again, for more info, check out [this](https://colah.github.io/posts/2015-08-Understanding-LSTMs/) excellent post. \n",
    "\n",
    "The rest of the encoder should be very familar from the last tutorial, it takes in a sequence, $X = \\{x_1, x_2, ... , x_T\\}$, passes it through the embedding layer, recurrently calculates hidden states, $H = \\{h_1, h_2, ..., h_T\\}$, and returns a context vector (the final hidden state), $z=h_T$.\n",
    "\n",
    "$$h_t = \\text{EncoderGRU}(e(x_t), h_{t-1})$$\n",
    "\n",
    "This is identical to the encoder of the general seq2seq model, with all the \"magic\" happening inside the GRU (green).\n",
    "\n",
    "![](assets/seq2seq5.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:46:34.198933Z",
     "iopub.status.busy": "2022-02-14T07:46:34.198791Z",
     "iopub.status.idle": "2022-02-14T07:46:34.203307Z",
     "shell.execute_reply": "2022-02-14T07:46:34.202898Z",
     "shell.execute_reply.started": "2022-02-14T07:46:34.198919Z"
    },
    "executionInfo": {
     "elapsed": 307,
     "status": "ok",
     "timestamp": 1644819943910,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "zabfsjZOUn3A",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, vocab_size, emb_dim, hidden_dim,\n",
    "                 pad_idx, emb_drop_prob, num_layers, \n",
    "                 ):\n",
    "      \n",
    "      super().__init__()\n",
    "\n",
    "      self.vocab_size = vocab_size\n",
    "      self.emb_dim = emb_dim\n",
    "      self.pad_idx= pad_idx\n",
    "      self.emb_drop_prob=emb_drop_prob\n",
    "      self.hidden_dim = hidden_dim\n",
    "      self.num_layers = num_layers\n",
    "      #self.rnn_drop_prob = rnn_drop_prob\n",
    "      \n",
    "      self.embedding = nn.Embedding(num_embeddings= self.vocab_size,\n",
    "                                        embedding_dim=self.emb_dim,\n",
    "                                        padding_idx=self.pad_idx)\n",
    "\n",
    "      self.dropout = nn.Dropout(p = self.emb_drop_prob)\n",
    "\n",
    "      self.gru_layer = nn.GRU(input_size=self.emb_dim,\n",
    "                               hidden_size=self.hidden_dim,\n",
    "                               num_layers=self.num_layers,\n",
    "                               batch_first=False,\n",
    "                               bidirectional=False,\n",
    "                               #dropout = self.rnn_drop_prob\n",
    "                                 )\n",
    "\n",
    "      \n",
    "        \n",
    "    def forward(self, source_indices):\n",
    "      # source_indices: [seq_len, batch_size]\n",
    "      emb = self.embedding(source_indices) # shape : [seq_len, batch_size, emb_dim]\n",
    "      emb_drop = self.dropout(emb) # shape : [seq_len, batch_size, emb_dim]\n",
    "      \n",
    "      \n",
    "      output, hidden = self.gru_layer(emb_drop ) # h0 is optional\n",
    "      # output : [seq_len, batch_size, directions * hidden_dim]\n",
    "      # hidden : [directions* num_layers, batch_size, hidden_dim]\n",
    "\n",
    "      return hidden\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7esX6rp0Un3A"
   },
   "source": [
    "## Decoder\n",
    "\n",
    "The decoder is where the implementation differs significantly from the previous model and we alleviate some of the information compression.\n",
    "\n",
    "Instead of the GRU in the decoder taking just the embedded target token, $d(y_t)$ and the previous hidden state $s_{t-1}$ as inputs, it also takes the context vector $z$. \n",
    "\n",
    "$$s_t = \\text{DecoderGRU}(d(y_t), s_{t-1}, z)$$\n",
    "\n",
    "Note how this context vector, $z$, does not have a $t$ subscript, meaning we re-use the same context vector returned by the encoder for every time-step in the decoder. \n",
    "\n",
    "Before, we predicted the next token, $\\hat{y}_{t+1}$, with the linear layer, $f$, only using the top-layer decoder hidden state at that time-step, $s_t$, as $\\hat{y}_{t+1}=f(s_t^L)$. Now, we also pass the embedding of current token, $d(y_t)$ and the context vector, $z$ to the linear layer.\n",
    "\n",
    "$$\\hat{y}_{t+1} = f(d(y_t), s_t, z)$$\n",
    "\n",
    "Thus, our decoder now looks something like this:\n",
    "\n",
    "![](assets/seq2seq6.png)\n",
    "\n",
    "Note, the initial hidden state, $s_0$, is still the context vector, $z$, so when generating the first token we are actually inputting two identical context vectors into the GRU.\n",
    "\n",
    "How do these two changes reduce the information compression? Well, hypothetically the decoder hidden states, $s_t$, no longer need to contain information about the source sequence as it is always available as an input. Thus, it only needs to contain information about what tokens it has generated so far. The addition of $y_t$ to the linear layer also means this layer can directly see what the token is, without having to get this information from the hidden state. \n",
    "\n",
    "However, this hypothesis is just a hypothesis, it is impossible to determine how the model actually uses the information provided to it (don't listen to anyone that says differently). Nevertheless, it is a solid intuition and the results seem to indicate that this modifications are a good idea!\n",
    "\n",
    "Within the implementation, we will pass $d(y_t)$ and $z$ to the GRU by concatenating them together, so the input dimensions to the GRU are now `emb_dim + hid_dim` (as context vector will be of size `hid_dim`). The linear layer will take $d(y_t), s_t$ and $z$ also by concatenating them together, hence the input dimensions are now `emb_dim + hid_dim*2`. We also don't pass a value of dropout to the GRU as it only uses a single layer.\n",
    "\n",
    "`forward` now takes a `context` argument. Inside of `forward`, we concatenate $y_t$ and $z$ as `emb_con` before feeding to the GRU, and we concatenate $d(y_t)$, $s_t$ and $z$ together as `output` before feeding it through the linear layer to receive our predictions, $\\hat{y}_{t+1}$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:46:34.748419Z",
     "iopub.status.busy": "2022-02-14T07:46:34.748278Z",
     "iopub.status.idle": "2022-02-14T07:46:34.754016Z",
     "shell.execute_reply": "2022-02-14T07:46:34.753495Z",
     "shell.execute_reply.started": "2022-02-14T07:46:34.748405Z"
    },
    "executionInfo": {
     "elapsed": 313,
     "status": "ok",
     "timestamp": 1644819957119,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "AVkx_S2fUn3A",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, vocab_size, emb_dim, hidden_dim, \n",
    "                 pad_idx, emb_drop_prob, num_layers, \n",
    "                 ):\n",
    "      \n",
    "     \n",
    "      super().__init__()\n",
    "\n",
    "      self.vocab_size = vocab_size\n",
    "      self.emb_dim = emb_dim\n",
    "      self.pad_idx= pad_idx\n",
    "      self.emb_drop_prob=emb_drop_prob\n",
    "      self.hidden_dim = hidden_dim\n",
    "      self.num_layers = num_layers\n",
    "      #self.rnn_drop_prob = rnn_drop_prob\n",
    "      \n",
    "\n",
    "      self.embedding = nn.Embedding(num_embeddings= self.vocab_size,\n",
    "                                        embedding_dim=self.emb_dim,\n",
    "                                        padding_idx=self.pad_idx)\n",
    "\n",
    "      self.dropout = nn.Dropout(p = self.emb_drop_prob) \n",
    "\n",
    "      self.gru_layer = nn.GRU(input_size=self.emb_dim + self.hidden_dim,\n",
    "                               hidden_size=self.hidden_dim,\n",
    "                               num_layers=self.num_layers,\n",
    "                               batch_first=False,\n",
    "                               bidirectional=False,\n",
    "                               #dropout = self.rnn_drop_prob\n",
    "                                 )\n",
    "      \n",
    "      self.linear= nn.Linear(in_features=self.hidden_dim + self.emb_dim + self.hidden_dim,\n",
    "                                 out_features=self.vocab_size)\n",
    "\n",
    "      \n",
    "        \n",
    "    def forward(self, input_token, hidden, hidden_src):\n",
    "\n",
    "      # in decoder we pass one token at a time- seq_len is 1\n",
    "      # shape of input_token : [batch_size]\n",
    "\n",
    "      #input_token = input_token.unsqueeze(0) # [1, batch_size, emb_dim]\n",
    "      #print(input_token.shape) \n",
    "            \n",
    "      emb = self.dropout(self.embedding(input_token)) \n",
    "      # emb - [batch_size, emb_dim]\n",
    "      \n",
    "      # lstm layer needs input in the shape: [seq_len, batch_size, emb_dim]\n",
    "      # we will add a redundant dimension to change  the shape of emb\n",
    "      emb = emb.unsqueeze(0)\n",
    "      \n",
    "      #print(emb.shape)\n",
    "      output, hidden = self.gru_layer(torch.cat((emb, hidden_src), dim = 2), hidden )\n",
    "\n",
    "      # output - [seq_len, batch_size, directions * hidden_dim]\n",
    "      # sequence length is always one for decoder as we pass one token at a time\n",
    "      # we never use bidirectional for decoder as decoder cannot look ahead\n",
    "      # hence shapes \n",
    "      # output: [1, batch_size, hidden_dim) \n",
    "      # hidden - [num_layers, batch_size, hidden_dim]\n",
    "      \n",
    "      # squeeze out the redundant dim from emd, output, enc_hidden to make it work for linear layer  \n",
    "      output = output.squeeze(0)\n",
    "      emb = emb.squeeze(0)\n",
    "      hidden_src = hidden_src.squeeze(0)\n",
    "        \n",
    "      # output: [1, batch_size, hidden_dim) \n",
    "      prediction = self.linear(torch.cat((output, emb, hidden_src), dim =1))\n",
    "      # prediction: [batch_size, decoder_vocab_size] \n",
    "      # each word is projected to vocab size\n",
    "      \n",
    "      return prediction, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VNxlQ5KYUn3A"
   },
   "source": [
    "## Seq2Seq Model\n",
    "\n",
    "Putting the encoder and decoder together, we get:\n",
    "\n",
    "![](assets/seq2seq7.png)\n",
    "\n",
    "Again, in this implementation we need to ensure the hidden dimensions in both the encoder and the decoder are the same.\n",
    "\n",
    "Briefly going over all of the steps:\n",
    "- the `outputs` tensor is created to hold all predictions, $\\hat{Y}$\n",
    "- the source sequence, $X$, is fed into the encoder to receive a `context` vector\n",
    "- the initial decoder hidden state is set to be the `context` vector, $s_0 = z = h_T$\n",
    "- we use a batch of `<sos>` tokens as the first `input`, $y_1$\n",
    "- we then decode within a loop:\n",
    "  - inserting the input token $y_t$, previous hidden state, $s_{t-1}$, and the context vector, $z$, into the decoder\n",
    "  - receiving a prediction, $\\hat{y}_{t+1}$, and a new hidden state, $s_t$\n",
    "  - we then decide if we are going to teacher force or not, setting the next input as appropriate (either the ground truth next token in the target sequence or the highest predicted next token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T07:46:35.543453Z",
     "iopub.status.busy": "2022-02-14T07:46:35.543312Z",
     "iopub.status.idle": "2022-02-14T07:46:35.547992Z",
     "shell.execute_reply": "2022-02-14T07:46:35.547694Z",
     "shell.execute_reply.started": "2022-02-14T07:46:35.543438Z"
    },
    "executionInfo": {
     "elapsed": 409,
     "status": "ok",
     "timestamp": 1644819962546,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "kH4m5zk7Un3B",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, encoder, decoder, device ):\n",
    "\n",
    "      super().__init__()\n",
    "\n",
    "      self.decoder = decoder\n",
    "      self.encoder = encoder\n",
    "      self.device = device\n",
    "      \n",
    "\n",
    "      assert self.decoder.hidden_dim == self.encoder.hidden_dim, \\\n",
    "      'The hidden_dim for encoder and decoder should be equal'\n",
    "\n",
    "      assert self.decoder.num_layers == self.encoder.num_layers, \\\n",
    "      ' The number of layers in encoder and decoder should be same'\n",
    "\n",
    "\n",
    "        \n",
    "    def forward(self, source_indices, target_indices, teacher_enf_ratio ):\n",
    "      # source_indices, target_indices # [seq_len, batch_size]\n",
    "      \n",
    "      \n",
    "\n",
    "      seq_len = target_indices.shape[0]\n",
    "      batch_size =  target_indices.shape[1]\n",
    "      vocab_size = self.decoder.vocab_size\n",
    "\n",
    "      predictions = torch.zeros((seq_len, batch_size, vocab_size)).to(self.device) # [tar_len, batch_size, target_vocab_size]\n",
    "\n",
    "      hidden_src = self.encoder(source_indices) # [directions * num_layer, batch_size, hidden_dim]\n",
    "    \n",
    "      input_token = target_indices[0,:] # [batch_size]\n",
    "      \n",
    "      hidden = hidden_src\n",
    "\n",
    "      # we will not update the predictions corresponding to first token -<BOS> \n",
    "\n",
    "      for i in range(1, len(target_indices)):\n",
    "        prediction, hidden = self.decoder(input_token, hidden, hidden_src) # prediction: [batch_size, decoder_vocab_size] \n",
    "\n",
    "        # update predictions\n",
    "        predictions[i] = prediction\n",
    "\n",
    "        teacher_force = torch.rand(1) < teacher_enf_ratio\n",
    "\n",
    "        if teacher_force:\n",
    "          input_token = target_indices[i,:]\n",
    "        else:\n",
    "          input_token = torch.argmax(prediction, dim =1) # batch_size\n",
    "\n",
    "      return predictions # [tar_len, batch_size, target_vocab_size]   \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HyPbOrlHUn3B"
   },
   "source": [
    "# Training the Seq2Seq Model\n",
    "\n",
    "The rest of this tutorial is very similar to the previous one. \n",
    "\n",
    "We initialise our encoder, decoder and seq2seq model (placing it on the GPU if we have one). As before, the embedding dimensions and the amount of dropout used can be different between the encoder and the decoder, but the hidden dimensions must remain the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:12.299141Z",
     "iopub.status.busy": "2022-02-14T08:04:12.298925Z",
     "iopub.status.idle": "2022-02-14T08:04:12.389170Z",
     "shell.execute_reply": "2022-02-14T08:04:12.388676Z",
     "shell.execute_reply.started": "2022-02-14T08:04:12.299107Z"
    },
    "executionInfo": {
     "elapsed": 10468,
     "status": "ok",
     "timestamp": 1644819978013,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "xg6tH3mDUn3B",
    "outputId": "d122a2f9-95c4-49b2-9cfc-d18a4d08b61c",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Seq2Seq(\n",
       "  (decoder): Decoder(\n",
       "    (embedding): Embedding(6537, 256, padding_idx=3)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "    (gru_layer): GRU(512, 256)\n",
       "    (linear): Linear(in_features=768, out_features=6537, bias=True)\n",
       "  )\n",
       "  (encoder): Encoder(\n",
       "    (embedding): Embedding(6115, 128, padding_idx=3)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "    (gru_layer): GRU(128, 256)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list of hyperparameters for encoder, decoder, model\n",
    "ENC_VOCAB_SIZE = len(source_vocab)\n",
    "ENC_EMB = 128\n",
    "HID_DIM = 256\n",
    "ENC_PAD_IDX = source_vocab['<PAD>']\n",
    "ENC_EMB_DROP_PROB = 0.5\n",
    "NUM_LAYERS = 1\n",
    "#ENC_RNN_DROP_PROB = 0.5\n",
    "\n",
    "DEC_VOCAB_SIZE = len(target_vocab)\n",
    "DEC_EMB = 256\n",
    "DEC_PAD_IDX = target_vocab['<PAD>']\n",
    "DEC_EMB_DROP_PROB = 0.5\n",
    "#DEC_RNN_DROP_PROB = 0.5\n",
    "\n",
    "LEARNING_RATE = 0.001\n",
    "\n",
    "enc = Encoder(vocab_size=ENC_VOCAB_SIZE , emb_dim =ENC_EMB, hidden_dim=HID_DIM,\n",
    "              pad_idx=ENC_PAD_IDX, emb_drop_prob=ENC_EMB_DROP_PROB, num_layers=NUM_LAYERS, \n",
    "              )\n",
    "\n",
    "dec = Decoder(vocab_size=DEC_VOCAB_SIZE, emb_dim=DEC_EMB, hidden_dim=HID_DIM, \n",
    "              pad_idx=DEC_PAD_IDX, emb_drop_prob=DEC_EMB_DROP_PROB, num_layers=NUM_LAYERS, \n",
    "              )\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "\n",
    "model = Seq2Seq(encoder=enc, decoder=dec, device=device )\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "R8QiCSrzUn3B"
   },
   "source": [
    "Next, we initialize our parameters. The paper states the parameters are initialized from a normal distribution with a mean of 0 and a standard deviation of 0.01, i.e. $\\mathcal{N}(0, 0.01)$. \n",
    "\n",
    "It also states we should initialize the recurrent parameters to a special initialization, however to keep things simple we'll also initialize them to $\\mathcal{N}(0, 0.01)$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:12.701843Z",
     "iopub.status.busy": "2022-02-14T08:04:12.701670Z",
     "iopub.status.idle": "2022-02-14T08:04:12.707585Z",
     "shell.execute_reply": "2022-02-14T08:04:12.707180Z",
     "shell.execute_reply.started": "2022-02-14T08:04:12.701822Z"
    },
    "executionInfo": {
     "elapsed": 198,
     "status": "ok",
     "timestamp": 1644819980356,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "8a9Kgdh5Un3B",
    "outputId": "51ea24ea-6254-4fa1-87fd-ac1cda7d1170",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Seq2Seq(\n",
       "  (decoder): Decoder(\n",
       "    (embedding): Embedding(6537, 256, padding_idx=3)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "    (gru_layer): GRU(512, 256)\n",
       "    (linear): Linear(in_features=768, out_features=6537, bias=True)\n",
       "  )\n",
       "  (encoder): Encoder(\n",
       "    (embedding): Embedding(6115, 128, padding_idx=3)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "    (gru_layer): GRU(128, 256)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_weights(m):\n",
    "    for name, param in m.named_parameters():\n",
    "        nn.init.normal_(param.data, 0, 0.01)\n",
    "        \n",
    "model.apply(init_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kX8djfNtUn3B"
   },
   "source": [
    "We also define a function that will calculate the number of trainable parameters in the model.\n",
    "\n",
    "Even though we only have a single layer RNN for our encoder and decoder we actually have **more** parameters  than the last model. This is due to the increased size of the inputs to the GRU and the linear layer. However, it is not a significant amount of parameters and causes a minimal amount of increase in training time (~3 seconds per epoch extra)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:13.109905Z",
     "iopub.status.busy": "2022-02-14T08:04:13.109688Z",
     "iopub.status.idle": "2022-02-14T08:04:13.113553Z",
     "shell.execute_reply": "2022-02-14T08:04:13.113236Z",
     "shell.execute_reply.started": "2022-02-14T08:04:13.109870Z"
    },
    "executionInfo": {
     "elapsed": 447,
     "status": "ok",
     "timestamp": 1644819983987,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "SJW-AD6NUn3C",
    "outputId": "331ac24b-8ca8-44ed-9e6f-bdd5b0fcf300",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 8,370,953 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f'The model has {count_parameters(model):,} trainable parameters')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kqFRDWy5Un3C"
   },
   "source": [
    "We define our optimizer, which we use to update our parameters in the training loop. Check out [this](http://ruder.io/optimizing-gradient-descent/) post for information about different optimizers. Here, we'll use Adam."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:13.542660Z",
     "iopub.status.busy": "2022-02-14T08:04:13.542455Z",
     "iopub.status.idle": "2022-02-14T08:04:13.545460Z",
     "shell.execute_reply": "2022-02-14T08:04:13.545114Z",
     "shell.execute_reply.started": "2022-02-14T08:04:13.542627Z"
    },
    "executionInfo": {
     "elapsed": 308,
     "status": "ok",
     "timestamp": 1644819986509,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "uQW8MwmpUn3C",
    "tags": []
   },
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vP8-nNUgUn3C"
   },
   "source": [
    "Next, we define our loss function. The `CrossEntropyLoss` function calculates both the log softmax as well as the negative log-likelihood of our predictions. \n",
    "\n",
    "Our loss function calculates the average loss per token, however by passing the index of the `<pad>` token as the `ignore_index` argument we ignore the loss whenever the target token is a padding token. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:13.975539Z",
     "iopub.status.busy": "2022-02-14T08:04:13.975354Z",
     "iopub.status.idle": "2022-02-14T08:04:13.978312Z",
     "shell.execute_reply": "2022-02-14T08:04:13.977858Z",
     "shell.execute_reply.started": "2022-02-14T08:04:13.975520Z"
    },
    "executionInfo": {
     "elapsed": 220,
     "status": "ok",
     "timestamp": 1644819988571,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "HqDGxXU-2A7y",
    "tags": []
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss(ignore_index=DEC_PAD_IDX)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DTSE8TjRUn3C"
   },
   "source": [
    "Next, we'll define our training loop. \n",
    "\n",
    "First, we'll set the model into \"training mode\" with `model.train()`. This will turn on dropout (and batch normalization, which we aren't using) and then iterate through our data iterator.\n",
    "\n",
    "As stated before, our decoder loop starts at 1, not 0. This means the 0th element of our `outputs` tensor remains all zeros. So our `trg` and `outputs` look something like:\n",
    "\n",
    "$$\\begin{align*}\n",
    "\\text{trg} = [<sos>, &y_1, y_2, y_3, <eos>]\\\\\n",
    "\\text{outputs} = [0, &\\hat{y}_1, \\hat{y}_2, \\hat{y}_3, <eos>]\n",
    "\\end{align*}$$\n",
    "\n",
    "Here, when we calculate the loss, we cut off the first element of each tensor to get:\n",
    "\n",
    "$$\\begin{align*}\n",
    "\\text{trg} = [&y_1, y_2, y_3, <eos>]\\\\\n",
    "\\text{outputs} = [&\\hat{y}_1, \\hat{y}_2, \\hat{y}_3, <eos>]\n",
    "\\end{align*}$$\n",
    "\n",
    "At each iteration:\n",
    "- get the source and target sentences from the batch, $X$ and $Y$\n",
    "- zero the gradients calculated from the last batch\n",
    "- feed the source and target into the model to get the output, $\\hat{Y}$\n",
    "- as the loss function only works on 2d inputs with 1d targets we need to flatten each of them with `.view`\n",
    "    - we slice off the first column of the output and target tensors as mentioned above\n",
    "- calculate the gradients with `loss.backward()`\n",
    "- clip the gradients to prevent them from exploding (a common issue in RNNs)\n",
    "- update the parameters of our model by doing an optimizer step\n",
    "- sum the loss value to a running total\n",
    "\n",
    "Finally, we return the loss that is averaged over all batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:14.366522Z",
     "iopub.status.busy": "2022-02-14T08:04:14.366383Z",
     "iopub.status.idle": "2022-02-14T08:04:14.370963Z",
     "shell.execute_reply": "2022-02-14T08:04:14.370680Z",
     "shell.execute_reply.started": "2022-02-14T08:04:14.366509Z"
    },
    "executionInfo": {
     "elapsed": 308,
     "status": "ok",
     "timestamp": 1644819992243,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "BYKiriabUn3C",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, clip, teacher_enf_ratio):\n",
    "\n",
    "  model.train()\n",
    "  epoch_loss = 0\n",
    "\n",
    "  for src , tgt in iterator:\n",
    "    src = src.to(device)\n",
    "    tgt = tgt.to(device)\n",
    "    \n",
    "\n",
    "    # get predictions\n",
    "    logits = model(src, tgt, teacher_enf_ratio)\n",
    "\n",
    "    tgt = tgt[1:,:] # first seq corresponds to token '<BOS>'\n",
    "    logits = logits[1:, :]\n",
    "\n",
    "    num_classes  = logits.shape[-1]\n",
    "    logits = logits.view(-1, num_classes) # [(trg_seq_length-1) * batch_size, num_classes]\n",
    "    tgt = tgt.view(-1) # [(trg_seq_length-1) * batch_size]\n",
    "\n",
    "    # set gradients to zero to avoid gradient accumulation from previous iterations\n",
    "    optimizer.zero_grad\n",
    "\n",
    "    # calculate loss\n",
    "    #with torch.cuda.amp.autocast():\n",
    "    loss = criterion(logits, tgt)\n",
    "\n",
    "    # calculate gradients\n",
    "    loss.backward()\n",
    "    #scaler.scale(loss).backward()\n",
    "\n",
    "    # clip gradients\n",
    "    clip_grad_norm_(model.parameters(), clip)\n",
    "\n",
    "    # update parameters\n",
    "    optimizer.step()\n",
    "    \n",
    "    #scaler.step(optimizer)\n",
    "    #scaler.update()\n",
    "    \n",
    "    epoch_loss+= loss.item()\n",
    "\n",
    "  return epoch_loss/len(iterator)   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XEEHX3lcUn3C"
   },
   "source": [
    "Our evaluation loop is similar to our training loop, however as we aren't updating any parameters we don't need to pass an optimizer or a clip value.\n",
    "\n",
    "We must remember to set the model to evaluation mode with `model.eval()`. This will turn off dropout (and batch normalization, if used).\n",
    "\n",
    "We use the `with torch.no_grad()` block to ensure no gradients are calculated within the block. This reduces memory consumption and speeds things up. \n",
    "\n",
    "The iteration loop is similar (without the parameter updates), however we must ensure we turn teacher forcing off for evaluation. This will cause the model to only use it's own predictions to make further predictions within a sentence, which mirrors how it would be used in deployment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:14.711385Z",
     "iopub.status.busy": "2022-02-14T08:04:14.711132Z",
     "iopub.status.idle": "2022-02-14T08:04:14.714987Z",
     "shell.execute_reply": "2022-02-14T08:04:14.714615Z",
     "shell.execute_reply.started": "2022-02-14T08:04:14.711369Z"
    },
    "executionInfo": {
     "elapsed": 263,
     "status": "ok",
     "timestamp": 1644819996339,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "q_0CMMb9Un3C",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion, teacher_enf_ratio):\n",
    "    model.eval()\n",
    "    epoch_loss = 0\n",
    "  \n",
    "\n",
    "    with torch.no_grad():\n",
    "      for src , tgt in iterator:\n",
    "\n",
    "        src = src.to(device) # [src_seq_len, batch_size]\n",
    "\n",
    "        tgt = tgt.to(device) # [tgt_seq_len, batch_size]\n",
    "\n",
    "        # get predictions\n",
    "        logits = model(src, tgt, teacher_enf_ratio) # [trg_seq_length, batch_size, tgt_vocab_size]\n",
    "\n",
    "        tgt = tgt[1:,:] # first seq corresponds to token '<BOS>'\n",
    "        logits = logits[1:, :]\n",
    "\n",
    "        num_classes  = logits.shape[-1]\n",
    "        logits = logits.view(-1, num_classes) # [(trg_seq_length-1) * batch_size, num_classes]\n",
    "        tgt = tgt.view(-1) # [(trg_seq_length-1) * batch_size]\n",
    "\n",
    "        # calculate loss\n",
    "        loss = criterion(logits, tgt)\n",
    "\n",
    "        epoch_loss+= loss.item()\n",
    "\n",
    "    return epoch_loss/len(iterator)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dBWvFyRmUn3D"
   },
   "source": [
    "Next, we'll create a function that we'll use to tell us how long an epoch takes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:15.010802Z",
     "iopub.status.busy": "2022-02-14T08:04:15.010660Z",
     "iopub.status.idle": "2022-02-14T08:04:15.013583Z",
     "shell.execute_reply": "2022-02-14T08:04:15.013166Z",
     "shell.execute_reply.started": "2022-02-14T08:04:15.010788Z"
    },
    "executionInfo": {
     "elapsed": 476,
     "status": "ok",
     "timestamp": 1644819999719,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "dwfIeh7XUn3D",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n16Yz_VcUn3D"
   },
   "source": [
    "We can finally start training our model!\n",
    "\n",
    "At each epoch, we'll be checking if our model has achieved the best validation loss so far. If it has, we'll update our best validation loss and save the parameters of our model (called `state_dict` in PyTorch). Then, when we come to test our model, we'll use the saved parameters used to achieve the best validation loss. \n",
    "\n",
    "We'll be printing out both the loss and the perplexity at each epoch. It is easier to see a change in perplexity than a change in loss as the numbers are much bigger."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:15.340348Z",
     "iopub.status.busy": "2022-02-14T08:04:15.340024Z",
     "iopub.status.idle": "2022-02-14T08:04:16.661289Z",
     "shell.execute_reply": "2022-02-14T08:04:16.660757Z",
     "shell.execute_reply.started": "2022-02-14T08:04:15.340332Z"
    },
    "executionInfo": {
     "elapsed": 187,
     "status": "ok",
     "timestamp": 1644820003446,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "vfV-uRNI2OCZ",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import  gc\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T08:04:16.662337Z",
     "iopub.status.busy": "2022-02-14T08:04:16.662235Z",
     "iopub.status.idle": "2022-02-14T08:22:13.006306Z",
     "shell.execute_reply": "2022-02-14T08:22:13.005923Z",
     "shell.execute_reply.started": "2022-02-14T08:04:16.662324Z"
    },
    "executionInfo": {
     "elapsed": 131293,
     "status": "ok",
     "timestamp": 1644820136541,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "7vyJuNQzUn3D",
    "outputId": "68e260c7-e812-457e-c49a-30a6a40bec2e",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Time: 1m 47s\n",
      "\tTrain Loss: 5.918 | Train PPL: 371.627\n",
      "\t Val. Loss: 5.873 |  Val. PPL: 355.267\n",
      "Epoch: 02 | Time: 1m 48s\n",
      "\tTrain Loss: 5.444 | Train PPL: 231.302\n",
      "\t Val. Loss: 5.871 |  Val. PPL: 354.501\n",
      "Epoch: 03 | Time: 1m 47s\n",
      "\tTrain Loss: 5.267 | Train PPL: 193.831\n",
      "\t Val. Loss: 5.815 |  Val. PPL: 335.439\n",
      "Epoch: 04 | Time: 1m 48s\n",
      "\tTrain Loss: 5.129 | Train PPL: 168.812\n",
      "\t Val. Loss: 5.786 |  Val. PPL: 325.801\n",
      "Epoch: 05 | Time: 1m 48s\n",
      "\tTrain Loss: 4.946 | Train PPL: 140.628\n",
      "\t Val. Loss: 5.700 |  Val. PPL: 298.742\n",
      "Epoch: 06 | Time: 1m 47s\n",
      "\tTrain Loss: 4.812 | Train PPL: 122.964\n",
      "\t Val. Loss: 5.618 |  Val. PPL: 275.373\n",
      "Epoch: 07 | Time: 1m 46s\n",
      "\tTrain Loss: 4.672 | Train PPL: 106.947\n",
      "\t Val. Loss: 5.558 |  Val. PPL: 259.415\n",
      "Epoch: 08 | Time: 1m 48s\n",
      "\tTrain Loss: 4.513 | Train PPL:  91.223\n",
      "\t Val. Loss: 5.512 |  Val. PPL: 247.537\n",
      "Epoch: 09 | Time: 1m 47s\n",
      "\tTrain Loss: 4.370 | Train PPL:  79.035\n",
      "\t Val. Loss: 5.487 |  Val. PPL: 241.552\n",
      "Epoch: 10 | Time: 1m 46s\n",
      "\tTrain Loss: 4.245 | Train PPL:  69.742\n",
      "\t Val. Loss: 5.466 |  Val. PPL: 236.416\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 10\n",
    "CLIP = 1\n",
    "TRAIN_TEACHER_ENF_RATIO = 0.5\n",
    "VALID_TEACHER_ENF_RATIO = 0 # turn off teacher enforcing for evaluation\n",
    "\n",
    "min_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss = train(model, train_loader, optimizer, criterion, CLIP, TRAIN_TEACHER_ENF_RATIO)\n",
    "    valid_loss = evaluate(model, valid_loader, criterion, VALID_TEACHER_ENF_RATIO)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    \n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    # save the model corresponding to min vlaid loss\n",
    "    if valid_loss<min_valid_loss:\n",
    "      min_valid_loss = valid_loss\n",
    "      torch.save(model.state_dict(), folder/'2b_en_hi.pt')\n",
    "    \n",
    "   \n",
    "    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UeXMknRvUn3D"
   },
   "source": [
    "We'll load the parameters (`state_dict`) that gave our model the best validation loss and run it the model on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2022-02-14T08:22:13.007161Z",
     "iopub.status.busy": "2022-02-14T08:22:13.006886Z",
     "iopub.status.idle": "2022-02-14T08:22:14.332460Z",
     "shell.execute_reply": "2022-02-14T08:22:14.332074Z",
     "shell.execute_reply.started": "2022-02-14T08:22:13.007143Z"
    },
    "executionInfo": {
     "elapsed": 638,
     "status": "ok",
     "timestamp": 1644563889501,
     "user": {
      "displayName": "Shaannoor Mann",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "02520257695567980696"
     },
     "user_tz": 360
    },
    "id": "YFz9w3KmUn3D",
    "outputId": "61a64a2b-fc6e-4a97-de33-5f8cc4cbfd5f",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Test Loss: 5.491 | Test PPL: 242.552 |\n"
     ]
    }
   ],
   "source": [
    "# load the saved model and get test loss\n",
    "model.load_state_dict(torch.load(folder/'2b_en_hi.pt'))\n",
    "test_loss = evaluate(model, test_loader, criterion, 0)\n",
    "print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3d61-wekUn3D"
   },
   "source": [
    "In the following notebook we'll implement a model that achieves improved test perplexity, but only uses a single layer in the encoder and the decoder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "2c - En-Hi_Sequence to Sequence Learning with Neural Networks-Copy1.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "pt10",
   "language": "python",
   "name": "pt10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}