Transformers_implementation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 350,
      "metadata": {
        "id": "m6iCzARbf1SR"
      },
      "outputs": [],
      "source": [
        "# Attention is all you need paper - https://arxiv.org/pdf/1706.03762.pdf\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optimizer\n",
        "import math\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class ScaledDotProductAtt(nn.Module):\n",
        "  def __init__(self, dropout=0.1): #dropout as input for avoiding overfitting and improve generalization\n",
        "    super(ScaledDotProductAtt, self).__init__()\n",
        "\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "  def forward(self, query, key, value, mask=None): #optional mask\n",
        "    attScores = torch.matmul(query, key.transpose(-2, -1)) / np.sqrt(key.size(-1))\n",
        "    if mask is not None: #Mask usually in the decoder\n",
        "      attScores = attScores.masked_fill(mask == 0, -1e10) #Avoiding mask to be 0 to avoid inestabilities, low number instead\n",
        "\n",
        "    attention = F.softmax(attScores, dim = -1)#last axis\n",
        "    attention = self.dropout(attention) #dropping 30% of the scores\n",
        "    return torch.matmul(attention, value), attention #added attention to get what the model is attending to"
      ],
      "metadata": {
        "id": "cq7mg-f0gJbg"
      },
      "execution_count": 351,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MultiHeadAttention(nn.Module):\n",
        "  def __init__(self, d_model, nhead, dropout=0.1):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.d_model = d_model # model dimension\n",
        "    self.nhead = nhead # number of \"heads\"\n",
        "    self.d_k = d_model // nhead # key dimension\n",
        "    self.d_v = d_model // nhead # value dimension\n",
        "\n",
        "    # Linearity for inputs\n",
        "    self.linear_q = nn.Linear(d_model, d_model)\n",
        "    self.linear_k = nn.Linear(d_model, d_model)\n",
        "    self.linear_v = nn.Linear(d_model, d_model)\n",
        "\n",
        "    self.scaledDotProductAttention = ScaledDotProductAtt(dropout)\n",
        "\n",
        "    self.linearLayer = nn.Linear(d_model, d_model) # Linear layer at output\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "  def forward(self, query, key, value, mask=None, key_padding_mask=None):\n",
        "    batchSize = query.size(0)\n",
        "\n",
        "    query = self.linear_q(query).view(batchSize, -1, self.nhead, self.d_k).transpose(1,2)\n",
        "    key = self.linear_q(key).view(batchSize, -1, self.nhead, self.d_k).transpose(1,2)\n",
        "    value = self.linear_q(value).view(batchSize, -1, self.nhead, self.d_v).transpose(1,2)\n",
        "\n",
        "    output, attScores = self.scaledDotProductAttention(query, key, value)\n",
        "\n",
        "    outputConcat = output.transpose(1,2).contiguous().view(batchSize, -1, self.d_model)\n",
        "    outputConcat = self.linearLayer(outputConcat)\n",
        "\n",
        "    return self.dropout(outputConcat)"
      ],
      "metadata": {
        "id": "3FSXekWfAtM0"
      },
      "execution_count": 352,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "  def __init__(self, d_model, dropout=0.1, maxLength=100):\n",
        "    super(PositionalEncoding, self).__init__()\n",
        "\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    pe = torch.zeros(maxLength, d_model)\n",
        "    position = torch.arange(0, maxLength, dtype=torch.float).unsqueeze(1)\n",
        "    divisionTerm = torch.exp(torch.arange(0, d_model, 2).float() * -(torch.log(torch.tensor(10000.0)) / d_model))\n",
        "\n",
        "    pe[:, 0::2] = torch.sin(position * divisionTerm)\n",
        "    pe[:, 1::2] = torch.cos(position * divisionTerm)\n",
        "\n",
        "    pe = pe.unsqueeze(0).transpose(0,1)\n",
        "    self.register_buffer('pe', pe)\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = x + self.pe[:x.size(0), :]\n",
        "    return self.dropout(x)"
      ],
      "metadata": {
        "id": "UU1apcHrFAI4"
      },
      "execution_count": 353,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class FeedForward(nn.Module):\n",
        "  def __init__(self, d_model, d_mlp=1024, dropout=0.1):\n",
        "    super(FeedForward, self).__init__()\n",
        "    self.linear_1 = nn.Linear(d_model, d_mlp)\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    self.linear_2 = nn.Linear(d_mlp, d_model)\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = self.linear_1(x)\n",
        "    x = F.relu(x)\n",
        "    x = self.dropout(x)\n",
        "    x = self.linear_2(x)\n",
        "\n",
        "    return x"
      ],
      "metadata": {
        "id": "Ki2AmAX6JApG"
      },
      "execution_count": 354,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class NormalizationLayer(nn.Module):\n",
        "  def __init__(self, d_model, epsilon=1e-5):\n",
        "    super(NormalizationLayer, self).__init__()\n",
        "    self.gamma = nn.Parameter(torch.ones(d_model))\n",
        "    self.beta = nn.Parameter(torch.zeros(d_model))\n",
        "    self.epsilon = epsilon\n",
        "\n",
        "  def forward(self, x):\n",
        "    mean = x.mean(dim=1, keepdim=True)\n",
        "    std = x.std(dim=-1, keepdim=True)\n",
        "\n",
        "    x = (x - mean) / (std + self.epsilon)\n",
        "    x = self.gamma * x + self.beta\n",
        "\n",
        "    return x"
      ],
      "metadata": {
        "id": "mgwp1CmsKQGQ"
      },
      "execution_count": 355,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Encoder(nn.Module):\n",
        "  def __init__(self, d_model, nhead, d_mlp, dropout=0.1):\n",
        "    super(Encoder, self).__init__()\n",
        "\n",
        "    self.multiHeadAttention = MultiHeadAttention(d_model, nhead, dropout)\n",
        "\n",
        "    self.feedforward = FeedForward(d_model, d_mlp, dropout)\n",
        "\n",
        "    self.normLayer1 = NormalizationLayer(d_model)\n",
        "    self.normLayer2 = NormalizationLayer(d_model)\n",
        "\n",
        "    self.dropout1 = nn.Dropout(dropout)\n",
        "    self.dropout2 = nn.Dropout(dropout)\n",
        "\n",
        "  def forward(self, x, src_mask=None, src_key_padding_mask=None, is_causal=False):\n",
        "    x2 = self.multiHeadAttention(x, x, x, mask=src_mask, key_padding_mask=src_key_padding_mask)[0]\n",
        "    x2 = self.normLayer1(x2)\n",
        "\n",
        "    x = x + self.dropout1(x2)\n",
        "\n",
        "    x2 = self.feedforward(x)\n",
        "    x2 = self.normLayer2(x2)\n",
        "    x = x + self.dropout2(x2)\n",
        "\n",
        "    return x\n",
        "\n"
      ],
      "metadata": {
        "id": "HlVNXM0mLsx_"
      },
      "execution_count": 356,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Decoder(nn.Module):\n",
        "  def __init__(self, d_model, nhead, d_mlp, dropout=0.1):\n",
        "    super(Decoder, self).__init__()\n",
        "\n",
        "    self.maskedMultiHeadAttention = MultiHeadAttention(d_model, nhead, dropout)\n",
        "    self.multiHeadAttention = MultiHeadAttention(d_model, nhead, dropout)\n",
        "\n",
        "    self.feedforward = FeedForward(d_model, d_mlp, dropout)\n",
        "\n",
        "    self.normLayer1 = NormalizationLayer(d_model)\n",
        "    self.normLayer2 = NormalizationLayer(d_model)\n",
        "    self.normLayer3 = NormalizationLayer(d_model)\n",
        "\n",
        "    self.dropout1 = nn.Dropout(dropout)\n",
        "    self.dropout2 = nn.Dropout(dropout)\n",
        "    self.dropout3 = nn.Dropout(dropout)\n",
        "\n",
        "  def forward(self, target, memory, tgt_mask=None, memory_mask=None, tgt_key_padding_mask=None, memory_key_padding_mask=None):\n",
        "    target2 = self.maskedMultiHeadAttention(target, target, target, mask=tgt_mask, key_padding_mask=tgt_key_padding_mask)[0]\n",
        "    target2 = self.normLayer1(target2)\n",
        "    target = target + self.dropout1(target2)\n",
        "\n",
        "    target2 = self.multiHeadAttention(target2, memory, memory, mask=memory_mask, key_padding_mask=memory_key_padding_mask)[0]\n",
        "    target2 = self.normLayer2(target2)\n",
        "    target = target + self.dropout2(target2)\n",
        "\n",
        "    target2 = self.feedforward(target)\n",
        "    target2 = self.normLayer3(target2)\n",
        "    target = target + self.dropout3(target2)\n",
        "\n",
        "    return target"
      ],
      "metadata": {
        "id": "-UIGGonNPVzE"
      },
      "execution_count": 357,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Transformer(nn.Module):\n",
        "  def __init__(self, d_model, nhead, nEncoder, nDecoder, d_mlp, maxLength, nChar, padIndex, dropout=0.1):\n",
        "    super(Transformer, self).__init__()\n",
        "    self.d_model = d_model\n",
        "\n",
        "    encoderLayer = Encoder(d_model, nhead, d_mlp, dropout)\n",
        "    encoderNorm = NormalizationLayer(d_model)\n",
        "    self.encoder = nn.TransformerEncoder(encoderLayer, nEncoder, encoderNorm)\n",
        "\n",
        "    decoderLayer = Decoder(d_model, nhead, d_mlp, dropout)\n",
        "    decoderNorm = NormalizationLayer(d_model)\n",
        "    self.decoder = nn.TransformerDecoder(decoderLayer, nDecoder, decoderNorm)\n",
        "\n",
        "    self.posEncoder = PositionalEncoding(d_model, dropout, maxLength)\n",
        "\n",
        "    self.inputEmbed = nn.Embedding(nChar, d_model, padding_idx=padIndex)\n",
        "    self.outputEmbed = nn.Embedding(nChar, d_model, padding_idx=padIndex)\n",
        "\n",
        "    self.linear = nn.Linear(d_model, nChar)\n",
        "\n",
        "  def forward(self, src, output, src_mask=None, outputMask=None, src_key_padding_mask=None, output_keyPaddingMask=None, memory_keyPaddingMask=None, isCausal=False):\n",
        "    src = self.inputEmbed(src) * np.sqrt(self.d_model)\n",
        "    src = self.posEncoder(src)\n",
        "    encoderOutputs = self.encoder(src, mask=src_mask, src_key_padding_mask=src_key_padding_mask, is_causal=isCausal)\n",
        "\n",
        "    output = self.outputEmbed(output) * np.sqrt(self.d_model)\n",
        "    output = self.posEncoder(output)\n",
        "    decoderOutputs = self.decoder(output, encoderOutputs, tgt_mask=outputMask, memory_mask=None, tgt_key_padding_mask=output_keyPaddingMask, memory_key_padding_mask=memory_keyPaddingMask)\n",
        "\n",
        "    outputs = self.linear(decoderOutputs)\n",
        "    return outputs\n"
      ],
      "metadata": {
        "id": "TF8NYtuNSelt"
      },
      "execution_count": 358,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "d_model = 512\n",
        "nhead = 1\n",
        "nEncoder = 1\n",
        "nDecoder = 1\n",
        "d_mlp = 1024\n",
        "maxLength = 6\n",
        "nChar = 26\n",
        "padIndex = 0\n",
        "dropout = 0.1"
      ],
      "metadata": {
        "id": "HpHt4wZVX8rt"
      },
      "execution_count": 359,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Transformer(d_model, nhead, nEncoder, nDecoder, d_mlp, maxLength, nChar, padIndex, dropout)"
      ],
      "metadata": {
        "id": "kfpGMMM9YeOK"
      },
      "execution_count": 360,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import Dataset, DataLoader"
      ],
      "metadata": {
        "id": "Yt7xOVAc6R39"
      },
      "execution_count": 361,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class ReverseDS(Dataset):\n",
        "  def __init__(self, length=10000, seqLength=10):\n",
        "    self.length = length\n",
        "    self.seqLenght = seqLength\n",
        "    self.vocab = list(\"abcdefghijklmnopqrstuvwxyz\")\n",
        "    self.vocabSize = len(self.vocab)\n",
        "    self.charToIdx = {char: idx for idx, char in enumerate(self.vocab)}\n",
        "    self.idxToChar = {idx: char for idx, char in enumerate(self.vocab)}\n",
        "\n",
        "  def __len__(self):\n",
        "    return self.length\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    sequence = torch.randint(high=self.vocabSize, size=(self.seqLenght,))\n",
        "    return sequence, torch.flip(sequence, dims=[0])"
      ],
      "metadata": {
        "id": "lffCvQFu6ul_"
      },
      "execution_count": 362,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = ReverseDS(seqLength=maxLength)\n",
        "dataloader = DataLoader(dataset, batch_size=5, shuffle=True)"
      ],
      "metadata": {
        "id": "AWfbDUG59KIg"
      },
      "execution_count": 363,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model = Transformer(d_model, nhead, nEncoder, nDecoder, d_mlp, maxLength, nChar, padIndex, dropout).to(device)\n",
        "\n",
        "loss_func = nn.CrossEntropyLoss()\n",
        "optimize = optimizer.Adam(model.parameters(), lr=0.0001)"
      ],
      "metadata": {
        "id": "vNU9NnMe-Bhu"
      },
      "execution_count": 364,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tokensToText(tokens, dataset):\n",
        "  return ''.join(dataset.idxToChar[token.item()] for token in tokens)"
      ],
      "metadata": {
        "id": "X0ddCyW5-wN_"
      },
      "execution_count": 365,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs, targets = next(iter(dataloader))\n",
        "print(\"input: \", tokensToText(inputs[4], dataset))\n",
        "print(\"target: \", tokensToText(targets[4], dataset))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pgNmAOWx_b3b",
        "outputId": "54780dfd-5590-46d6-89d8-06b187d9ff55"
      },
      "execution_count": 366,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input:  dobqsc\n",
            "target:  csqbod\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Training Loop\n",
        "nEpochs = 100\n",
        "\n",
        "for epoch in range(nEpochs):\n",
        "  for i, (input, target) in enumerate(dataloader):\n",
        "    input = input.T.to(device)\n",
        "    target = target.T.to(device)\n",
        "\n",
        "    target_input = target[:-1, :]\n",
        "    target_real = target[1:, :]\n",
        "\n",
        "    output = model(input, target_real)\n",
        "\n",
        "    lossFunc = loss_func(output.view(-1, nChar), target_real.reshape(-1))\n",
        "    optimize.zero_grad()\n",
        "    lossFunc.backward()\n",
        "    optimize.step()\n",
        "\n",
        "    if i % 100 == 0:\n",
        "      print(f\"Epoch: {epoch}, Iteration: {i}, Loss: {lossFunc.item()}\")\n",
        "      break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w3ytt1AHBYcL",
        "outputId": "815b2006-aa0e-46c7-caad-e31ea299ed3e"
      },
      "execution_count": 443,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0, Iteration: 0, Loss: 0.035629939287900925\n",
            "Epoch: 1, Iteration: 0, Loss: 0.021889066323637962\n",
            "Epoch: 2, Iteration: 0, Loss: 0.05668865516781807\n",
            "Epoch: 3, Iteration: 0, Loss: 0.04098549485206604\n",
            "Epoch: 4, Iteration: 0, Loss: 0.03635813295841217\n",
            "Epoch: 5, Iteration: 0, Loss: 0.047168828547000885\n",
            "Epoch: 6, Iteration: 0, Loss: 0.055058594793081284\n",
            "Epoch: 7, Iteration: 0, Loss: 0.03191569447517395\n",
            "Epoch: 8, Iteration: 0, Loss: 0.030601775273680687\n",
            "Epoch: 9, Iteration: 0, Loss: 0.04115985706448555\n",
            "Epoch: 10, Iteration: 0, Loss: 0.030507758259773254\n",
            "Epoch: 11, Iteration: 0, Loss: 0.030679047107696533\n",
            "Epoch: 12, Iteration: 0, Loss: 0.04582683742046356\n",
            "Epoch: 13, Iteration: 0, Loss: 0.09573394060134888\n",
            "Epoch: 14, Iteration: 0, Loss: 0.026687031611800194\n",
            "Epoch: 15, Iteration: 0, Loss: 0.0627489984035492\n",
            "Epoch: 16, Iteration: 0, Loss: 0.04570617526769638\n",
            "Epoch: 17, Iteration: 0, Loss: 0.0775856226682663\n",
            "Epoch: 18, Iteration: 0, Loss: 0.015062820166349411\n",
            "Epoch: 19, Iteration: 0, Loss: 0.13924597203731537\n",
            "Epoch: 20, Iteration: 0, Loss: 0.038678500801324844\n",
            "Epoch: 21, Iteration: 0, Loss: 0.033618759363889694\n",
            "Epoch: 22, Iteration: 0, Loss: 0.04457961395382881\n",
            "Epoch: 23, Iteration: 0, Loss: 0.04704755172133446\n",
            "Epoch: 24, Iteration: 0, Loss: 0.05067342892289162\n",
            "Epoch: 25, Iteration: 0, Loss: 0.040329836308956146\n",
            "Epoch: 26, Iteration: 0, Loss: 0.023154601454734802\n",
            "Epoch: 27, Iteration: 0, Loss: 0.02712344564497471\n",
            "Epoch: 28, Iteration: 0, Loss: 0.03416038677096367\n",
            "Epoch: 29, Iteration: 0, Loss: 0.03824144974350929\n",
            "Epoch: 30, Iteration: 0, Loss: 0.038223106414079666\n",
            "Epoch: 31, Iteration: 0, Loss: 0.03777875378727913\n",
            "Epoch: 32, Iteration: 0, Loss: 0.02528836578130722\n",
            "Epoch: 33, Iteration: 0, Loss: 0.04865718260407448\n",
            "Epoch: 34, Iteration: 0, Loss: 0.04682183265686035\n",
            "Epoch: 35, Iteration: 0, Loss: 0.01986750401556492\n",
            "Epoch: 36, Iteration: 0, Loss: 0.04330416023731232\n",
            "Epoch: 37, Iteration: 0, Loss: 0.10008427500724792\n",
            "Epoch: 38, Iteration: 0, Loss: 0.031449850648641586\n",
            "Epoch: 39, Iteration: 0, Loss: 0.03698873892426491\n",
            "Epoch: 40, Iteration: 0, Loss: 0.050989218056201935\n",
            "Epoch: 41, Iteration: 0, Loss: 0.029613591730594635\n",
            "Epoch: 42, Iteration: 0, Loss: 0.029023896902799606\n",
            "Epoch: 43, Iteration: 0, Loss: 0.035261787474155426\n",
            "Epoch: 44, Iteration: 0, Loss: 0.03487256169319153\n",
            "Epoch: 45, Iteration: 0, Loss: 0.027970563620328903\n",
            "Epoch: 46, Iteration: 0, Loss: 0.04055508226156235\n",
            "Epoch: 47, Iteration: 0, Loss: 0.03834743797779083\n",
            "Epoch: 48, Iteration: 0, Loss: 0.03305823728442192\n",
            "Epoch: 49, Iteration: 0, Loss: 0.02767171338200569\n",
            "Epoch: 50, Iteration: 0, Loss: 0.019209058955311775\n",
            "Epoch: 51, Iteration: 0, Loss: 0.04016141593456268\n",
            "Epoch: 52, Iteration: 0, Loss: 0.03971591591835022\n",
            "Epoch: 53, Iteration: 0, Loss: 0.02076944336295128\n",
            "Epoch: 54, Iteration: 0, Loss: 0.05885213240981102\n",
            "Epoch: 55, Iteration: 0, Loss: 0.03444221615791321\n",
            "Epoch: 56, Iteration: 0, Loss: 0.03585472330451012\n",
            "Epoch: 57, Iteration: 0, Loss: 0.053244587033987045\n",
            "Epoch: 58, Iteration: 0, Loss: 0.054011646658182144\n",
            "Epoch: 59, Iteration: 0, Loss: 0.05085098370909691\n",
            "Epoch: 60, Iteration: 0, Loss: 0.03206853196024895\n",
            "Epoch: 61, Iteration: 0, Loss: 0.04349363222718239\n",
            "Epoch: 62, Iteration: 0, Loss: 0.033619269728660583\n",
            "Epoch: 63, Iteration: 0, Loss: 0.02030331641435623\n",
            "Epoch: 64, Iteration: 0, Loss: 0.031194813549518585\n",
            "Epoch: 65, Iteration: 0, Loss: 0.05079512670636177\n",
            "Epoch: 66, Iteration: 0, Loss: 0.023410160094499588\n",
            "Epoch: 67, Iteration: 0, Loss: 0.040004849433898926\n",
            "Epoch: 68, Iteration: 0, Loss: 0.04604185000061989\n",
            "Epoch: 69, Iteration: 0, Loss: 0.05309279263019562\n",
            "Epoch: 70, Iteration: 0, Loss: 0.04219513386487961\n",
            "Epoch: 71, Iteration: 0, Loss: 0.0339728482067585\n",
            "Epoch: 72, Iteration: 0, Loss: 0.04918798804283142\n",
            "Epoch: 73, Iteration: 0, Loss: 0.02557198703289032\n",
            "Epoch: 74, Iteration: 0, Loss: 0.09062041342258453\n",
            "Epoch: 75, Iteration: 0, Loss: 0.027601394802331924\n",
            "Epoch: 76, Iteration: 0, Loss: 0.034782811999320984\n",
            "Epoch: 77, Iteration: 0, Loss: 0.02067895419895649\n",
            "Epoch: 78, Iteration: 0, Loss: 0.03607974201440811\n",
            "Epoch: 79, Iteration: 0, Loss: 0.02715597301721573\n",
            "Epoch: 80, Iteration: 0, Loss: 0.03974210470914841\n",
            "Epoch: 81, Iteration: 0, Loss: 0.01900332048535347\n",
            "Epoch: 82, Iteration: 0, Loss: 0.03981759399175644\n",
            "Epoch: 83, Iteration: 0, Loss: 0.022652599960565567\n",
            "Epoch: 84, Iteration: 0, Loss: 0.016458241268992424\n",
            "Epoch: 85, Iteration: 0, Loss: 0.061833757907152176\n",
            "Epoch: 86, Iteration: 0, Loss: 0.027274668216705322\n",
            "Epoch: 87, Iteration: 0, Loss: 0.03519006073474884\n",
            "Epoch: 88, Iteration: 0, Loss: 0.029665803536772728\n",
            "Epoch: 89, Iteration: 0, Loss: 0.03631730005145073\n",
            "Epoch: 90, Iteration: 0, Loss: 0.049408216029405594\n",
            "Epoch: 91, Iteration: 0, Loss: 0.02966180071234703\n",
            "Epoch: 92, Iteration: 0, Loss: 0.055797018110752106\n",
            "Epoch: 93, Iteration: 0, Loss: 0.044593773782253265\n",
            "Epoch: 94, Iteration: 0, Loss: 0.039067938923835754\n",
            "Epoch: 95, Iteration: 0, Loss: 0.04784338176250458\n",
            "Epoch: 96, Iteration: 0, Loss: 0.035451874136924744\n",
            "Epoch: 97, Iteration: 0, Loss: 0.06831184029579163\n",
            "Epoch: 98, Iteration: 0, Loss: 0.05072057247161865\n",
            "Epoch: 99, Iteration: 0, Loss: 0.0447414293885231\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def outputToText(output, dataset):\n",
        "  tokens = F.softmax(output, dim=-1)\n",
        "  tokens = torch.argmax(tokens, dim=-1)\n",
        "  text = ''.join(dataset.idxToChar[token.item()] for token in tokens)\n",
        "  return text"
      ],
      "metadata": {
        "id": "Jur60k7uz3iJ"
      },
      "execution_count": 444,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs, targets = next(iter(dataloader))\n",
        "index = 1\n",
        "print(\"input: \", tokensToText(inputs[index], dataset))\n",
        "print(\"target: \", tokensToText(targets[index], dataset))\n",
        "\n",
        "input = inputs[index].T.to(device)\n",
        "target = targets[index].T.to(device)\n",
        "print(target)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oijNOIgl0mXA",
        "outputId": "bcc43015-ce1b-4534-d44e-b6d0eaabda6f"
      },
      "execution_count": 445,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input:  anwfqd\n",
            "target:  dqfwna\n",
            "tensor([ 3, 16,  5, 22, 13,  0])\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-445-1f025e7262a3>:6: UserWarning: The use of `x.T` on tensors of dimension other than 2 to reverse their shape is deprecated and it will throw an error in a future release. Consider `x.mT` to transpose batches of matrices or `x.permute(*torch.arange(x.ndim - 1, -1, -1))` to reverse the dimensions of a tensor. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3571.)\n",
            "  input = inputs[index].T.to(device)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output = model(input, target)\n",
        "print(output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lC_x2wcw1URG",
        "outputId": "ad90ad29-2c66-4db8-c432-de2af9e4cf57"
      },
      "execution_count": 446,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[[-5.6854e+00, -4.0797e-01, -8.5232e-01,  8.6193e+00,  2.2450e-01,\n",
            "           1.0304e+00,  1.7942e-01, -7.5684e-02, -6.8202e-02, -3.4973e-01,\n",
            "           6.3008e-02, -7.2342e-01, -4.1481e-01,  1.6703e-01,  4.7294e-02,\n",
            "           1.2958e-01,  1.0995e+00,  1.5599e-01, -1.1579e-01, -4.4391e-01,\n",
            "          -4.0762e-01, -3.0254e-01,  6.8252e-01, -4.9980e-01,  2.7178e-01,\n",
            "          -9.6120e-01],\n",
            "         [-6.0419e+00, -1.2720e+00, -5.4319e-01,  1.3245e-02,  5.9138e-01,\n",
            "           1.6527e+00, -4.9871e-01, -3.0150e-01, -4.6245e-01, -3.5863e-01,\n",
            "          -3.9353e-01, -4.5832e-01, -1.4129e-01,  4.0501e-01, -2.2501e-01,\n",
            "          -9.6693e-02,  9.0548e+00, -1.0426e+00, -4.1564e-01, -8.4929e-01,\n",
            "           8.2666e-01, -1.7150e+00, -1.6670e-01, -3.7460e-01,  2.0122e-01,\n",
            "          -5.4441e-01],\n",
            "         [-6.5338e+00,  1.2563e-01, -1.2479e+00,  4.8221e-02,  7.7787e-01,\n",
            "           9.2884e+00,  1.4282e-01, -9.6761e-01, -6.1540e-01, -9.0445e-01,\n",
            "          -2.9012e-01, -7.6176e-01,  3.9338e-01, -1.1119e+00, -6.0449e-01,\n",
            "           1.3826e-02,  1.3187e+00, -6.5298e-01,  3.8658e-02, -1.1358e+00,\n",
            "          -4.1843e-01, -7.8615e-01,  3.2132e-01,  3.1966e-01, -6.4491e-01,\n",
            "          -1.0425e+00],\n",
            "         [-5.3403e+00, -8.4077e-02, -1.7732e-01, -2.6542e-02,  8.9773e-01,\n",
            "           7.8310e-01, -3.2358e-01, -5.8343e-01, -2.4380e-02,  2.1303e-01,\n",
            "           2.4314e-01, -3.4473e-01, -2.4589e-01,  2.0955e-01,  5.1715e-01,\n",
            "           2.2575e-01,  1.0194e+00, -3.7421e-03, -7.9897e-01, -1.2145e-01,\n",
            "          -2.6675e-01, -6.7197e-01,  9.2918e+00, -6.8510e-01, -2.1894e-01,\n",
            "           1.4640e-01],\n",
            "         [-6.2636e+00, -1.5419e-01, -1.2285e-01,  5.1785e-01, -4.2854e-03,\n",
            "           6.8215e-01, -1.0841e+00, -3.0605e-01, -2.1052e-01, -9.1448e-02,\n",
            "          -6.8644e-01, -3.1490e-01,  7.4338e-01,  8.3774e+00, -1.0611e+00,\n",
            "           8.5592e-01,  1.0401e+00, -6.2284e-01, -3.0447e-01,  1.5875e-01,\n",
            "          -7.5071e-01, -5.0978e-01,  8.2438e-01, -5.1126e-01, -8.1373e-02,\n",
            "           1.3476e-01],\n",
            "         [ 1.0840e+01,  7.8273e-01,  1.1289e+00, -3.5059e+00, -8.5449e-01,\n",
            "          -5.0018e+00,  5.2270e-01,  7.5334e-01,  5.2885e-01,  6.1916e-01,\n",
            "           4.5232e-01,  9.4124e-01, -8.0897e-02, -3.1199e+00,  5.4288e-01,\n",
            "          -3.7037e-01, -4.9139e+00,  7.4624e-01,  6.0993e-01,  9.3982e-01,\n",
            "           4.0651e-01,  1.4818e+00, -4.0709e+00,  5.2563e-01,  1.8229e-01,\n",
            "           9.3836e-01]],\n",
            "\n",
            "        [[-5.0347e+00, -4.9645e-02, -3.8416e-01,  8.4483e+00, -2.0305e-01,\n",
            "           1.1959e+00,  4.4081e-01, -1.7355e-01,  3.8667e-01, -2.5027e-01,\n",
            "           3.3170e-01, -1.2117e+00, -6.4324e-01, -2.5957e-01,  8.0884e-02,\n",
            "           2.4023e-02,  1.2569e+00,  3.6026e-01,  3.4877e-01, -1.2856e-01,\n",
            "          -2.0306e-01,  1.2314e-01,  7.1765e-02, -8.2624e-01,  2.2413e-01,\n",
            "          -1.2579e+00],\n",
            "         [-5.9053e+00, -7.4347e-01, -9.6128e-01,  1.2671e-01,  4.3261e-01,\n",
            "           1.2623e+00,  2.7209e-01, -2.2282e-01, -1.5454e-01, -7.8958e-02,\n",
            "          -4.5054e-01, -5.4244e-01, -2.3203e-01,  2.6918e-01, -6.1379e-01,\n",
            "           4.8008e-03,  8.7675e+00, -8.4363e-01, -3.7658e-01, -1.2846e+00,\n",
            "          -3.2840e-02, -1.2551e+00, -8.7193e-01, -5.8335e-02,  7.6319e-02,\n",
            "          -2.8204e-01],\n",
            "         [-6.6103e+00, -2.4834e-01, -1.5898e+00,  3.3675e-01,  5.3688e-01,\n",
            "           9.0590e+00,  5.6251e-01, -1.0984e+00,  1.3408e-02, -5.8809e-01,\n",
            "          -2.6118e-01, -7.6951e-01,  7.3354e-02, -1.0196e+00, -9.4341e-01,\n",
            "           2.0140e-01,  8.3074e-01, -1.1563e-02,  1.1598e-01, -7.6575e-01,\n",
            "          -4.1036e-01, -1.2695e+00,  3.4113e-01,  9.7450e-02, -7.0240e-01,\n",
            "          -9.7290e-01],\n",
            "         [-5.3814e+00, -1.1348e-01, -3.8754e-01,  4.1278e-01,  1.2982e+00,\n",
            "           4.5446e-01,  3.4408e-02, -4.2214e-01, -4.8961e-01,  3.6623e-01,\n",
            "           2.0665e-01, -3.6020e-01, -5.8325e-01,  2.4741e-01,  5.9726e-01,\n",
            "           2.6228e-01,  6.8310e-01,  1.9754e-01, -2.1795e-01, -4.3013e-03,\n",
            "          -1.1297e-01,  9.1314e-02,  8.7515e+00, -7.6068e-01, -8.0870e-04,\n",
            "          -1.4260e-02],\n",
            "         [-5.8283e+00,  2.4989e-02, -2.9558e-01, -2.1086e-01, -1.5229e-01,\n",
            "           4.0917e-01, -7.1762e-01, -2.0313e-01, -5.8156e-01, -1.6594e-01,\n",
            "          -4.3351e-01, -5.6459e-01,  5.9399e-01,  8.6456e+00, -9.1039e-01,\n",
            "           1.1130e+00,  8.3856e-01,  1.1633e-02, -3.0862e-01,  4.3884e-02,\n",
            "          -1.1291e+00, -7.9854e-01,  7.4194e-01, -2.8786e-01, -2.5308e-01,\n",
            "          -7.2206e-02],\n",
            "         [ 1.0861e+01,  5.5251e-01,  1.4272e+00, -3.5738e+00, -6.6167e-01,\n",
            "          -4.7923e+00, -2.9106e-01,  7.4070e-01,  3.4110e-01,  3.4724e-01,\n",
            "           2.9244e-01,  1.3134e+00,  3.4599e-01, -3.2211e+00,  7.4463e-01,\n",
            "          -5.7412e-01, -4.7092e+00,  5.3224e-02,  1.8814e-01,  8.8163e-01,\n",
            "           7.6108e-01,  1.2099e+00, -3.4801e+00,  5.8053e-01,  2.6371e-01,\n",
            "           1.1056e+00]],\n",
            "\n",
            "        [[-5.5177e+00,  1.5101e-01, -8.3078e-01,  8.4239e+00, -2.2638e-01,\n",
            "           8.8384e-01,  7.6719e-02, -4.7540e-01,  2.3401e-01, -5.1878e-03,\n",
            "          -2.9388e-01, -7.9207e-01, -2.5679e-01, -6.7726e-01,  5.5280e-01,\n",
            "          -2.3114e-01,  1.1383e+00,  1.8983e-01,  4.7435e-01, -1.1390e+00,\n",
            "          -2.3641e-01, -1.4851e-01,  5.8395e-01, -2.1750e-01,  5.0808e-02,\n",
            "          -1.2892e+00],\n",
            "         [-5.7708e+00, -5.4629e-01, -7.6118e-01,  3.0720e-01, -2.3014e-02,\n",
            "           6.5917e-01, -1.6780e-01, -1.8147e-02, -2.8484e-02, -4.8913e-01,\n",
            "          -4.4672e-01, -4.7385e-01, -2.0618e-01,  3.4040e-01, -3.5811e-02,\n",
            "           8.2910e-02,  8.7955e+00, -1.1227e+00, -9.5301e-02, -1.4657e+00,\n",
            "           5.4562e-01, -1.9411e+00, -1.3481e-01,  2.3947e-01,  4.1450e-01,\n",
            "          -8.5957e-01],\n",
            "         [-6.4510e+00,  2.5937e-01, -1.7858e+00, -1.2331e-01,  5.1775e-01,\n",
            "           8.7310e+00,  1.0745e-01, -8.7022e-01,  3.1419e-01, -8.1829e-01,\n",
            "          -3.2976e-01, -1.0872e+00,  6.1337e-01, -1.3990e+00, -9.1747e-01,\n",
            "           4.1847e-01,  1.6254e-01, -2.2068e-01, -5.2576e-01, -1.3966e+00,\n",
            "          -2.1395e-01, -9.6954e-01,  7.1121e-01,  1.2406e-01, -5.4799e-01,\n",
            "          -9.5700e-01],\n",
            "         [-5.0262e+00,  3.0506e-01, -2.6610e-01, -1.8336e-01,  4.4715e-01,\n",
            "           2.4642e-01, -7.7116e-01, -1.7670e-01, -4.9957e-01,  1.4520e-01,\n",
            "           9.3725e-02, -4.8376e-01,  7.0987e-02,  1.6223e-01,  3.7538e-01,\n",
            "           3.8382e-01,  9.6435e-01, -3.0710e-01, -1.6697e-01, -2.0852e-01,\n",
            "           4.5272e-01, -2.4244e-02,  9.1292e+00, -9.4552e-01, -1.3307e-01,\n",
            "          -4.1091e-02],\n",
            "         [-5.5888e+00,  2.8648e-01, -1.9700e-01,  1.5635e-01, -5.7171e-01,\n",
            "           3.9987e-01, -7.4803e-01,  3.6066e-01, -5.7323e-01, -5.6979e-02,\n",
            "          -1.7002e-01, -6.8808e-01,  1.0208e+00,  7.9975e+00, -9.3437e-01,\n",
            "           3.7249e-01,  8.3705e-01,  1.7611e-01, -2.5493e-01,  4.5752e-02,\n",
            "          -6.5504e-01,  1.0004e-01,  5.9664e-01,  1.2769e-01, -1.0632e-01,\n",
            "           1.8039e-01],\n",
            "         [ 1.0722e+01, -6.5636e-02,  1.5165e+00, -3.4623e+00,  2.2527e-02,\n",
            "          -4.2230e+00,  5.1438e-01,  3.8087e-01,  2.3781e-01,  5.3137e-01,\n",
            "           5.0039e-01,  1.3414e+00, -4.3738e-01, -2.6165e+00,  4.0743e-01,\n",
            "          -3.3271e-01, -4.4736e+00,  4.2566e-01,  2.2566e-01,  1.6677e+00,\n",
            "           7.2828e-02,  1.1395e+00, -4.2426e+00,  1.2769e-01,  1.3718e-01,\n",
            "           1.2508e+00]],\n",
            "\n",
            "        [[-5.6935e+00,  1.8502e-01, -8.6835e-01,  9.1094e+00, -8.0860e-02,\n",
            "           1.7255e+00,  5.2216e-01, -2.2657e-02, -3.1599e-01, -5.3420e-01,\n",
            "          -5.9307e-01, -5.8452e-01, -9.7178e-01, -6.2128e-01, -2.9763e-01,\n",
            "           1.1064e-01,  1.1503e+00,  1.2761e-01, -1.3024e-01, -8.2399e-01,\n",
            "          -1.1642e+00,  1.2701e-01,  1.0114e+00, -4.8414e-01,  2.8853e-01,\n",
            "          -1.7407e+00],\n",
            "         [-6.7249e+00, -9.1710e-01, -7.0492e-01,  5.9712e-01, -3.8121e-02,\n",
            "           1.4660e+00, -7.2774e-03,  7.4591e-02,  2.1293e-01, -1.0529e-01,\n",
            "          -3.9053e-01, -4.3537e-02,  3.3947e-02, -4.8647e-01, -3.9218e-01,\n",
            "          -2.2734e-01,  9.2019e+00, -1.0297e+00, -3.0314e-01, -1.4020e+00,\n",
            "           9.1383e-04, -1.3767e+00,  1.5024e-01, -2.9915e-01,  2.9274e-01,\n",
            "          -6.3128e-01],\n",
            "         [-7.5744e+00,  2.2872e-01, -8.2271e-01,  1.0516e+00,  4.6007e-01,\n",
            "           9.7614e+00,  6.9052e-01, -8.4769e-01, -6.1106e-02, -1.5693e+00,\n",
            "          -5.3600e-02, -9.6193e-01,  4.1673e-01, -1.0877e+00, -1.1420e+00,\n",
            "           2.0682e-01,  1.8288e-01, -2.8686e-02, -5.2238e-01, -1.0256e+00,\n",
            "          -6.8262e-01, -4.6925e-01,  6.4972e-01, -3.3636e-01, -5.8421e-01,\n",
            "          -5.9199e-01],\n",
            "         [-5.8711e+00, -2.6177e-01, -3.4747e-01,  3.7780e-01,  1.0524e+00,\n",
            "           1.0983e+00, -1.4518e-01, -6.3370e-01, -1.4000e-01,  2.8413e-01,\n",
            "          -8.6122e-03, -6.5173e-01,  3.7405e-01,  9.1075e-02,  4.2033e-01,\n",
            "           1.3686e-02,  1.1823e+00,  4.0376e-01, -4.9012e-01, -4.1759e-01,\n",
            "          -2.2394e-01, -6.4339e-01,  9.2833e+00, -3.8415e-01, -5.1220e-01,\n",
            "          -5.2715e-01],\n",
            "         [-6.2937e+00,  6.2275e-01, -2.2462e-01,  8.1739e-01, -4.6048e-01,\n",
            "           1.0365e+00, -9.1412e-01, -1.4492e-01, -6.0248e-01,  8.0628e-02,\n",
            "          -8.1496e-02,  6.7783e-03,  9.9410e-01,  7.9412e+00, -1.5922e+00,\n",
            "           8.7677e-01,  1.6598e+00, -3.5051e-01, -4.0369e-01, -6.1774e-02,\n",
            "          -1.5997e+00, -4.9268e-01,  1.0295e+00, -2.7926e-01,  4.1623e-01,\n",
            "           2.0532e-01],\n",
            "         [ 1.0406e+01,  1.4510e-01,  1.0206e+00, -4.0968e+00, -2.3933e-01,\n",
            "          -4.9879e+00, -1.1080e-01,  4.4694e-01,  3.2752e-01,  6.6666e-01,\n",
            "           4.3337e-01,  7.1105e-01, -2.2407e-01, -2.0346e+00,  1.0414e+00,\n",
            "          -2.7785e-01, -4.3490e+00,  2.3927e-01,  6.2598e-01,  1.2894e+00,\n",
            "           1.2555e+00,  9.4298e-01, -4.0227e+00,  4.7103e-01,  3.1011e-02,\n",
            "           1.1953e+00]],\n",
            "\n",
            "        [[-4.5696e+00,  1.9985e-01, -6.7725e-01,  8.8036e+00, -3.7010e-01,\n",
            "           9.7546e-01,  2.7989e-01,  1.4826e-02,  2.3755e-01,  2.4370e-01,\n",
            "           3.5773e-01, -6.8278e-01, -9.4395e-01, -7.0185e-01, -1.2501e-02,\n",
            "           4.8569e-02,  1.0486e+00,  3.4132e-01,  4.1365e-01, -5.2517e-01,\n",
            "          -4.2831e-01, -1.4453e-01,  5.6142e-01, -5.6263e-01, -6.8507e-02,\n",
            "          -1.2694e+00],\n",
            "         [-5.4538e+00, -1.0278e+00, -3.1166e-01, -2.3454e-01,  4.7278e-01,\n",
            "           6.5907e-01, -5.1726e-01, -1.8527e-01, -4.3031e-01,  2.8857e-01,\n",
            "          -1.9126e-01, -2.8685e-01, -4.4665e-01,  1.1385e-01,  6.2301e-02,\n",
            "          -1.2629e-01,  8.8048e+00, -7.3742e-01, -8.9810e-02, -1.0058e+00,\n",
            "           3.8956e-01, -1.5477e+00, -3.8386e-01, -5.4810e-01,  1.0015e-01,\n",
            "          -3.3907e-01],\n",
            "         [-6.1748e+00, -2.3325e-01, -1.0231e+00,  5.6422e-02,  1.0703e+00,\n",
            "           8.6166e+00,  9.9617e-01, -9.4134e-01, -1.1042e-01, -9.1184e-01,\n",
            "          -3.5397e-01, -6.2627e-01,  1.8192e-01, -1.4400e+00, -8.2110e-01,\n",
            "           4.0297e-01,  4.6838e-01, -8.8137e-02,  1.3314e-01, -1.0334e+00,\n",
            "          -6.9146e-01, -8.0912e-01,  2.4921e-01, -2.7549e-02, -1.0670e+00,\n",
            "          -8.4853e-01],\n",
            "         [-4.3913e+00, -2.6173e-01, -2.2917e-01,  1.0854e-01,  5.1483e-01,\n",
            "           1.3305e-01, -7.1419e-01, -6.8237e-01, -3.6329e-01,  8.9148e-01,\n",
            "           3.1802e-01, -4.9533e-01, -5.1231e-01, -4.4453e-01,  9.1716e-01,\n",
            "           1.2608e-01,  3.9273e-01,  4.3959e-01, -3.6736e-02, -1.0489e-01,\n",
            "          -3.4477e-01, -5.0682e-02,  8.7012e+00, -3.9088e-01, -6.6168e-02,\n",
            "          -2.8934e-01],\n",
            "         [-5.5027e+00,  1.5034e-01, -3.3344e-02,  2.0013e-01, -6.1946e-01,\n",
            "          -3.5645e-01, -1.0492e+00, -1.2963e-01, -2.3406e-01,  4.9408e-01,\n",
            "           1.5672e-01, -2.3109e-01,  9.3817e-01,  7.9776e+00, -8.6977e-01,\n",
            "           2.3078e-01,  6.1825e-01,  4.8118e-02, -7.0958e-01,  4.5191e-01,\n",
            "          -1.2286e+00, -3.5251e-01,  2.4042e-01,  7.5685e-02, -1.7728e-01,\n",
            "           1.1150e-01],\n",
            "         [ 1.0785e+01,  6.0784e-01,  1.0043e+00, -3.8823e+00, -3.6329e-01,\n",
            "          -4.2978e+00,  3.5358e-01,  7.3214e-01,  3.9709e-01, -3.5835e-01,\n",
            "          -6.9480e-02,  9.5690e-01,  3.7252e-01, -2.4918e+00,  3.7109e-01,\n",
            "          -2.2919e-01, -4.6584e+00, -6.8063e-02,  1.4538e-01,  9.9307e-01,\n",
            "           1.0164e+00,  1.2317e+00, -3.9571e+00,  4.6920e-01,  5.5819e-01,\n",
            "           1.2270e+00]],\n",
            "\n",
            "        [[-4.7150e+00, -5.0274e-02, -4.9794e-01,  8.8478e+00, -6.1317e-02,\n",
            "           8.5287e-01,  1.3355e-01,  3.9558e-02,  2.7366e-01, -7.3703e-02,\n",
            "           1.4112e-01, -9.2421e-01, -5.9910e-01, -2.1770e-01, -2.9006e-01,\n",
            "           6.5397e-02,  1.2624e+00,  4.1513e-01, -1.4973e-01, -4.5966e-01,\n",
            "          -9.4903e-01, -3.1489e-01,  7.7723e-01, -5.2969e-01,  2.7538e-01,\n",
            "          -9.1177e-01],\n",
            "         [-5.2927e+00, -7.8978e-01, -4.1030e-01, -1.8730e-01,  3.7922e-01,\n",
            "           6.1625e-01, -1.3876e-01, -1.2193e-01, -8.2867e-02, -3.3891e-01,\n",
            "          -5.5591e-02, -1.4848e-01, -1.4295e-01,  1.1547e-01, -2.3979e-01,\n",
            "          -1.6681e-01,  8.9819e+00, -4.4048e-01, -5.3331e-01, -1.2319e+00,\n",
            "          -5.2943e-02, -1.2829e+00, -6.9386e-01, -1.9330e-01,  6.0448e-01,\n",
            "          -2.7524e-01],\n",
            "         [-6.2671e+00, -1.4315e-01, -9.6130e-01,  8.1699e-02,  9.1362e-01,\n",
            "           8.5878e+00,  1.0145e+00, -5.8226e-01, -4.4868e-01, -8.8367e-01,\n",
            "          -5.5773e-01, -2.2279e-01,  1.2144e-01, -1.0603e+00, -7.1612e-01,\n",
            "          -6.0575e-02,  8.2525e-01,  8.3755e-02, -5.8950e-01, -8.7931e-01,\n",
            "          -8.2489e-01, -6.5523e-01,  3.1944e-01,  7.3809e-02, -5.9525e-01,\n",
            "          -9.6058e-01],\n",
            "         [-4.8794e+00, -1.5461e-01, -1.5120e-01, -1.0585e-01,  8.0981e-01,\n",
            "           9.0036e-01,  4.8089e-02, -4.6476e-01, -6.1644e-01,  9.0737e-01,\n",
            "          -1.6813e-01, -7.5874e-01,  9.5178e-02, -6.1024e-03,  7.7632e-01,\n",
            "           6.6870e-03,  4.3897e-01,  3.8443e-01, -2.1396e-01, -7.2398e-02,\n",
            "          -9.7667e-02,  3.3017e-01,  8.6633e+00, -1.5297e-01, -9.5315e-01,\n",
            "          -2.4198e-01],\n",
            "         [-5.1387e+00,  4.0964e-02,  2.5009e-01,  5.2735e-02, -1.1780e-01,\n",
            "           2.4073e-01, -1.5349e+00,  3.4117e-01, -3.7399e-01, -4.1399e-01,\n",
            "          -1.5324e-01, -1.7982e-01,  9.4510e-01,  8.4604e+00, -1.3067e+00,\n",
            "           1.1479e+00,  1.3273e+00,  1.4277e-01, -1.7177e-02,  5.3661e-01,\n",
            "          -1.1325e+00, -4.6353e-01,  1.9051e-01, -5.3858e-01, -2.2664e-01,\n",
            "           3.9960e-01],\n",
            "         [ 1.0775e+01,  5.7953e-01,  7.7881e-01, -3.8591e+00, -7.2481e-01,\n",
            "          -4.6994e+00,  1.4285e-01,  2.4261e-01,  5.3469e-01,  4.0552e-01,\n",
            "           3.8359e-01,  9.2631e-01, -1.2083e-01, -3.1913e+00,  8.0546e-01,\n",
            "          -3.6737e-01, -5.2715e+00, -3.1696e-01,  6.4411e-01,  9.3820e-01,\n",
            "           1.3273e+00,  1.0081e+00, -3.9156e+00,  4.3484e-01,  3.8167e-01,\n",
            "           9.4528e-01]]], grad_fn=<ViewBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"input: \", tokensToText(inputs[index], dataset))\n",
        "print(\"Prediction: \", outputToText(output[index], dataset))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t6Yas4sV1iZk",
        "outputId": "0477195b-a7f6-41fd-ba22-133bfbdf9509"
      },
      "execution_count": 447,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input:  anwfqd\n",
            "Prediction:  dqfwna\n"
          ]
        }
      ]
    }
  ]
}