ELT 579 118781 - Exemplo da semana 4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "57563d1b",
   "metadata": {
    "id": "FOcv8kNtIuqQ",
    "papermill": {
     "duration": 0.013689,
     "end_time": "2024-10-03T15:48:31.808869",
     "exception": false,
     "start_time": "2024-10-03T15:48:31.795180",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "Sequence-to-Sequence Modeling with nn.Transformer and TorchText\n",
    "===============================================================\n",
    "\n",
    "![](https://github.com/pytorch/tutorials/blob/gh-pages/_static/img/transformer_architecture.jpg?raw=1)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b8e57d95",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:48:31.838331Z",
     "iopub.status.busy": "2024-10-03T15:48:31.837885Z",
     "iopub.status.idle": "2024-10-03T15:50:50.023567Z",
     "shell.execute_reply": "2024-10-03T15:50:50.022444Z"
    },
    "executionInfo": {
     "elapsed": 127976,
     "status": "ok",
     "timestamp": 1726534601124,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "6RW7qlwyJVzn",
    "outputId": "bb80f457-a3ac-4983-dd10-c582b0972ae0",
    "papermill": {
     "duration": 138.204261,
     "end_time": "2024-10-03T15:50:50.026282",
     "exception": false,
     "start_time": "2024-10-03T15:48:31.822021",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing installation: torch 2.4.0\r\n",
      "Uninstalling torch-2.4.0:\r\n",
      "  Successfully uninstalled torch-2.4.0\r\n",
      "\u001b[33mWARNING: Skipping torchtext as it is not installed.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mCollecting torch==2.0.0\r\n",
      "  Downloading torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl.metadata (24 kB)\r\n",
      "Collecting torchtext==0.15.1\r\n",
      "  Downloading torchtext-0.15.1-cp310-cp310-manylinux1_x86_64.whl.metadata (7.4 kB)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0) (3.15.1)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0) (4.12.2)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0) (1.13.3)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0) (3.3)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch==2.0.0) (3.1.4)\r\n",
      "Collecting nvidia-cuda-nvrtc-cu11==11.7.99 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\r\n",
      "Collecting nvidia-cuda-runtime-cu11==11.7.99 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-cuda-cupti-cu11==11.7.101 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-cudnn-cu11==8.5.0.96 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-cublas-cu11==11.10.3.66 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-cufft-cu11==10.9.0.58 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\r\n",
      "Collecting nvidia-curand-cu11==10.2.10.91 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-cusolver-cu11==11.4.0.1 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-cusparse-cu11==11.7.4.91 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\r\n",
      "Collecting nvidia-nccl-cu11==2.14.3 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\r\n",
      "Collecting nvidia-nvtx-cu11==11.7.91 (from torch==2.0.0)\r\n",
      "  Downloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\r\n",
      "Collecting triton==2.0.0 (from torch==2.0.0)\r\n",
      "  Downloading triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.0 kB)\r\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from torchtext==0.15.1) (4.66.4)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from torchtext==0.15.1) (2.32.3)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from torchtext==0.15.1) (1.26.4)\r\n",
      "Collecting torchdata==0.6.0 (from torchtext==0.15.1)\r\n",
      "  Downloading torchdata-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (919 bytes)\r\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch==2.0.0) (70.0.0)\r\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch==2.0.0) (0.43.0)\r\n",
      "Requirement already satisfied: urllib3>=1.25 in /opt/conda/lib/python3.10/site-packages (from torchdata==0.6.0->torchtext==0.15.1) (1.26.18)\r\n",
      "Collecting cmake (from triton==2.0.0->torch==2.0.0)\r\n",
      "  Downloading cmake-3.30.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.4 kB)\r\n",
      "Collecting lit (from triton==2.0.0->torch==2.0.0)\r\n",
      "  Downloading lit-18.1.8-py3-none-any.whl.metadata (2.5 kB)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch==2.0.0) (2.1.5)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->torchtext==0.15.1) (3.3.2)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->torchtext==0.15.1) (3.7)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->torchtext==0.15.1) (2024.8.30)\r\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/conda/lib/python3.10/site-packages (from sympy->torch==2.0.0) (1.3.0)\r\n",
      "Downloading torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl (619.9 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m619.9/619.9 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading torchtext-0.15.1-cp310-cp310-manylinux1_x86_64.whl (2.0 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m51.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m77.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m64.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m36.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux2014_x86_64.whl (168.4 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.4/168.4 MB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 MB\u001b[0m \u001b[31m27.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 MB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m173.2/173.2 MB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.1/177.1 MB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.6/98.6 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading torchdata-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.6 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.6/4.6 MB\u001b[0m \u001b[31m59.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.3 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.3/63.3 MB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading cmake-3.30.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (26.9 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.9/26.9 MB\u001b[0m \u001b[31m56.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading lit-18.1.8-py3-none-any.whl (96 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.4/96.4 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hInstalling collected packages: lit, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, cmake, nvidia-cusolver-cu11, nvidia-cudnn-cu11, triton, torch, torchdata, torchtext\r\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "pytorch-lightning 2.4.0 requires torch>=2.1.0, but you have torch 2.0.0 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0mSuccessfully installed cmake-3.30.4 lit-18.1.8 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 torch-2.0.0 torchdata-0.6.0 torchtext-0.15.1 triton-2.0.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip uninstall -y torch torchtext\n",
    "!pip install torch==2.0.0 torchtext==0.15.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bff8a6d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:50.190905Z",
     "iopub.status.busy": "2024-10-03T15:50:50.190443Z",
     "iopub.status.idle": "2024-10-03T15:50:54.386980Z",
     "shell.execute_reply": "2024-10-03T15:50:54.385821Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 4.28271,
     "end_time": "2024-10-03T15:50:54.389767",
     "exception": false,
     "start_time": "2024-10-03T15:50:50.107057",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchtext\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import torch.optim as optim\n",
    "import time\n",
    "\n",
    "import os\n",
    "import math\n",
    "\n",
    "# Defina o comprimento máximo das sequências\n",
    "MAX_LENGTH = 100\n",
    "\n",
    "# Defina o tokenizador\n",
    "tokenizer = get_tokenizer('basic_english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "32edf752",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:54.550542Z",
     "iopub.status.busy": "2024-10-03T15:50:54.549818Z",
     "iopub.status.idle": "2024-10-03T15:50:54.634312Z",
     "shell.execute_reply": "2024-10-03T15:50:54.633059Z"
    },
    "papermill": {
     "duration": 0.168427,
     "end_time": "2024-10-03T15:50:54.637541",
     "exception": false,
     "start_time": "2024-10-03T15:50:54.469114",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rodando na GPU: Tesla P100-PCIE-16GB\n"
     ]
    }
   ],
   "source": [
    "# Verificar se a GPU está disponível\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# Exibir a informação\n",
    "if device.type == 'cuda':\n",
    "    print(\"Rodando na GPU:\", torch.cuda.get_device_name(0))\n",
    "else:\n",
    "    print(\"Rodando na CPU\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "47767c99",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:54.800528Z",
     "iopub.status.busy": "2024-10-03T15:50:54.800088Z",
     "iopub.status.idle": "2024-10-03T15:50:54.805817Z",
     "shell.execute_reply": "2024-10-03T15:50:54.804728Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.091592,
     "end_time": "2024-10-03T15:50:54.808157",
     "exception": false,
     "start_time": "2024-10-03T15:50:54.716565",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para ler os dados de texto\n",
    "def load_wikitext(filepath):\n",
    "    with open(filepath, 'r', encoding='utf-8') as f:\n",
    "        return f.read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d4b4307b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:54.973066Z",
     "iopub.status.busy": "2024-10-03T15:50:54.972071Z",
     "iopub.status.idle": "2024-10-03T15:50:54.978176Z",
     "shell.execute_reply": "2024-10-03T15:50:54.977079Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.089513,
     "end_time": "2024-10-03T15:50:54.980453",
     "exception": false,
     "start_time": "2024-10-03T15:50:54.890940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Caminhos para os arquivos no Google Drive\n",
    "folder = '/kaggle/input/wikitext-2'\n",
    "train_file = os.path.join(folder, 'wiki.train.tokens')\n",
    "val_file = os.path.join(folder, 'wiki.valid.tokens')\n",
    "test_file = os.path.join(folder, 'wiki.test.tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "44e0812b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:55.141143Z",
     "iopub.status.busy": "2024-10-03T15:50:55.140725Z",
     "iopub.status.idle": "2024-10-03T15:50:55.377170Z",
     "shell.execute_reply": "2024-10-03T15:50:55.376149Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.31957,
     "end_time": "2024-10-03T15:50:55.379813",
     "exception": false,
     "start_time": "2024-10-03T15:50:55.060243",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Carregar os dados\n",
    "train_txt = load_wikitext(train_file)\n",
    "val_txt = load_wikitext(val_file)\n",
    "test_txt = load_wikitext(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "510b863f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:55.541627Z",
     "iopub.status.busy": "2024-10-03T15:50:55.541161Z",
     "iopub.status.idle": "2024-10-03T15:50:55.547810Z",
     "shell.execute_reply": "2024-10-03T15:50:55.546140Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.091882,
     "end_time": "2024-10-03T15:50:55.551491",
     "exception": false,
     "start_time": "2024-10-03T15:50:55.459609",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para truncar ou preencher tokens\n",
    "def truncate_or_pad(tokens, max_length):\n",
    "    if len(tokens) > max_length:\n",
    "        return tokens[:max_length]\n",
    "    else:\n",
    "        return tokens + ['<pad>'] * (max_length - len(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5f2bd596",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:55.788857Z",
     "iopub.status.busy": "2024-10-03T15:50:55.788084Z",
     "iopub.status.idle": "2024-10-03T15:50:57.965365Z",
     "shell.execute_reply": "2024-10-03T15:50:57.964263Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 2.276566,
     "end_time": "2024-10-03T15:50:57.968082",
     "exception": false,
     "start_time": "2024-10-03T15:50:55.691516",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Tokenizar e ajustar as sequências\n",
    "train_tokens = [truncate_or_pad(tokenizer(line), MAX_LENGTH) for line in train_txt]\n",
    "val_tokens = [truncate_or_pad(tokenizer(line), MAX_LENGTH) for line in val_txt]\n",
    "test_tokens = [truncate_or_pad(tokenizer(line), MAX_LENGTH) for line in test_txt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "087ebbe6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:58.128497Z",
     "iopub.status.busy": "2024-10-03T15:50:58.127748Z",
     "iopub.status.idle": "2024-10-03T15:50:58.133020Z",
     "shell.execute_reply": "2024-10-03T15:50:58.131897Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.088002,
     "end_time": "2024-10-03T15:50:58.135245",
     "exception": false,
     "start_time": "2024-10-03T15:50:58.047243",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Defina os tokens especiais\n",
    "specials = ['<unk>', '<sos>', '<eos>', '<pad>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "89b83e30",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:58.298619Z",
     "iopub.status.busy": "2024-10-03T15:50:58.298110Z",
     "iopub.status.idle": "2024-10-03T15:50:58.304423Z",
     "shell.execute_reply": "2024-10-03T15:50:58.303324Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.090669,
     "end_time": "2024-10-03T15:50:58.306818",
     "exception": false,
     "start_time": "2024-10-03T15:50:58.216149",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para construir o vocabulário com tokens especiais\n",
    "def build_vocab_with_specials(tokens_list, specials):\n",
    "    # Adicione tokens especiais ao vocabulário\n",
    "    vocab = build_vocab_from_iterator(tokens_list, specials=specials, min_freq=1)\n",
    "    vocab.set_default_index(vocab[\"<unk>\"])\n",
    "    return vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d7ffde61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:58.468860Z",
     "iopub.status.busy": "2024-10-03T15:50:58.468419Z",
     "iopub.status.idle": "2024-10-03T15:50:59.035867Z",
     "shell.execute_reply": "2024-10-03T15:50:59.034737Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.650606,
     "end_time": "2024-10-03T15:50:59.038474",
     "exception": false,
     "start_time": "2024-10-03T15:50:58.387868",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Criar o vocabulário com tokens especiais\n",
    "vocab = build_vocab_with_specials(train_tokens, specials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9305ba9a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:59.204673Z",
     "iopub.status.busy": "2024-10-03T15:50:59.203888Z",
     "iopub.status.idle": "2024-10-03T15:50:59.210336Z",
     "shell.execute_reply": "2024-10-03T15:50:59.209188Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.092811,
     "end_time": "2024-10-03T15:50:59.212739",
     "exception": false,
     "start_time": "2024-10-03T15:50:59.119928",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para converter tokens em índices usando o vocabulário\n",
    "def tokenize_and_numericalize(data, vocab):\n",
    "    # Converta tokens em índices inteiros e garanta que o tensor seja do tipo Long\n",
    "    return [torch.tensor([vocab[token] for token in tokenizer(line)], dtype=torch.long) for line in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "78a3fc16",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:50:59.376406Z",
     "iopub.status.busy": "2024-10-03T15:50:59.375656Z",
     "iopub.status.idle": "2024-10-03T15:51:05.051408Z",
     "shell.execute_reply": "2024-10-03T15:51:05.050415Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 5.760232,
     "end_time": "2024-10-03T15:51:05.054069",
     "exception": false,
     "start_time": "2024-10-03T15:50:59.293837",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Numeralizar os dados de treino, validação e teste\n",
    "train_data = tokenize_and_numericalize(train_txt, vocab)\n",
    "val_data = tokenize_and_numericalize(val_txt, vocab)\n",
    "test_data = tokenize_and_numericalize(test_txt, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ef90beb3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:05.216547Z",
     "iopub.status.busy": "2024-10-03T15:51:05.215840Z",
     "iopub.status.idle": "2024-10-03T15:51:05.222381Z",
     "shell.execute_reply": "2024-10-03T15:51:05.221219Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.090975,
     "end_time": "2024-10-03T15:51:05.224691",
     "exception": false,
     "start_time": "2024-10-03T15:51:05.133716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Definir parâmetros do modelo\n",
    "ntokens = len(vocab)  # O tamanho do vocabulário\n",
    "emsize = 200          # Dimensão do embedding\n",
    "nhid = 200            # Dimensão do feedforward network model\n",
    "nlayers = 2           # Número de camadas do TransformerEncoder\n",
    "nhead = 2             # Número de cabeças na multiheadattention\n",
    "dropout = 0.2         # Valor de dropout\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7b9cecf5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:05.387528Z",
     "iopub.status.busy": "2024-10-03T15:51:05.386494Z",
     "iopub.status.idle": "2024-10-03T15:51:05.397174Z",
     "shell.execute_reply": "2024-10-03T15:51:05.396149Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.095149,
     "end_time": "2024-10-03T15:51:05.399510",
     "exception": false,
     "start_time": "2024-10-03T15:51:05.304361",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Definir modelo Transformer\n",
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, ntokens, emsize, nhead, nhid, nlayers, dropout=0.2):\n",
    "        super(TransformerModel, self).__init__()\n",
    "        self.model_type = 'Transformer'\n",
    "        self.encoder = nn.Embedding(ntokens, emsize)\n",
    "        self.pos_encoder = nn.Sequential(\n",
    "            nn.Dropout(dropout)\n",
    "        )\n",
    "        encoder_layers = nn.TransformerEncoderLayer(emsize, nhead, nhid, dropout)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(encoder_layers, nlayers)\n",
    "        self.decoder = nn.Linear(emsize, ntokens)\n",
    "        self.init_weights()\n",
    "\n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.encoder.weight.data.uniform_(-initrange, initrange)\n",
    "        self.decoder.bias.data.zero_()\n",
    "        self.decoder.weight.data.uniform_(-initrange, initrange)\n",
    "\n",
    "    def forward(self, src):\n",
    "        src = self.encoder(src) * math.sqrt(emsize)\n",
    "        src = self.pos_encoder(src)\n",
    "        output = self.transformer_encoder(src)\n",
    "        output = self.decoder(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9392c452",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:05.564473Z",
     "iopub.status.busy": "2024-10-03T15:51:05.563583Z",
     "iopub.status.idle": "2024-10-03T15:51:06.640084Z",
     "shell.execute_reply": "2024-10-03T15:51:06.638926Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 1.164519,
     "end_time": "2024-10-03T15:51:06.642860",
     "exception": false,
     "start_time": "2024-10-03T15:51:05.478341",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Inicializar o modelo\n",
    "model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "37c2df13",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:06.805438Z",
     "iopub.status.busy": "2024-10-03T15:51:06.804386Z",
     "iopub.status.idle": "2024-10-03T15:51:07.465520Z",
     "shell.execute_reply": "2024-10-03T15:51:07.464375Z"
    },
    "executionInfo": {
     "elapsed": 10277,
     "status": "ok",
     "timestamp": 1726537995383,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "MJW_B4S3pXOL",
    "outputId": "2879851d-0f80-44c0-d414-6101a5ce2ce1",
    "papermill": {
     "duration": 0.746738,
     "end_time": "2024-10-03T15:51:07.469436",
     "exception": false,
     "start_time": "2024-10-03T15:51:06.722698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para dividir os dados em batches\n",
    "def batchify(data, bsz):\n",
    "    # Converte tokens em tensores e preenche com zeros\n",
    "    data = pad_sequence([item for item in data], batch_first=True)\n",
    "    nbatch = data.size(0) // bsz\n",
    "    data = data[:nbatch * bsz].view(bsz, -1).t().contiguous()\n",
    "    return data.to(device, dtype=torch.long)\n",
    "\n",
    "batch_size = 20\n",
    "train_data = batchify(train_data, batch_size)\n",
    "val_data = batchify(val_data, batch_size)\n",
    "test_data = batchify(test_data, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bde37970",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:07.632145Z",
     "iopub.status.busy": "2024-10-03T15:51:07.631385Z",
     "iopub.status.idle": "2024-10-03T15:51:07.640844Z",
     "shell.execute_reply": "2024-10-03T15:51:07.639757Z"
    },
    "papermill": {
     "duration": 0.09277,
     "end_time": "2024-10-03T15:51:07.643284",
     "exception": false,
     "start_time": "2024-10-03T15:51:07.550514",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Ajuste a função get_batch para garantir que os tensores estejam no mesmo dispositivo\n",
    "def get_batch(source, i, bptt, device):\n",
    "    data = source[i:i + bptt]\n",
    "    target = source[i + 1:i + 1 + bptt]  # O alvo é deslocado em uma posição\n",
    "    \n",
    "    # Se a sequência de dados for menor que o bptt, adicione padding\n",
    "    if len(data) < bptt:\n",
    "        padding_len = bptt - len(data)\n",
    "        # Criar o padding e mover para o mesmo dispositivo do 'data'\n",
    "        padding = torch.full((padding_len, data.size(1)), vocab['<pad>'], dtype=torch.long).to(device)\n",
    "        data = torch.cat([data, padding], dim=0)\n",
    "\n",
    "    if len(target) < bptt:\n",
    "        padding_len = bptt - len(target)\n",
    "        # Criar o padding e mover para o mesmo dispositivo do 'target'\n",
    "        padding = torch.full((padding_len, target.size(1)), vocab['<pad>'], dtype=torch.long).to(device)\n",
    "        target = torch.cat([target, padding], dim=0)\n",
    "    \n",
    "    return data.to(device), target.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6e10125c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:07.804857Z",
     "iopub.status.busy": "2024-10-03T15:51:07.803855Z",
     "iopub.status.idle": "2024-10-03T15:51:07.811990Z",
     "shell.execute_reply": "2024-10-03T15:51:07.810816Z"
    },
    "papermill": {
     "duration": 0.092109,
     "end_time": "2024-10-03T15:51:07.814242",
     "exception": false,
     "start_time": "2024-10-03T15:51:07.722133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para verificar e ajustar os formatos\n",
    "def check_and_adjust_batch(data, target, model, criterion, device):\n",
    "    # Mover os tensores para o dispositivo correto (CPU ou GPU)\n",
    "    data = data.to(device)\n",
    "    target = target.to(device)\n",
    "    model = model.to(device)\n",
    "\n",
    "    # Checar formatos\n",
    "    print(f\"Data shape: {data.shape}\")\n",
    "    print(f\"Target shape: {target.shape}\")\n",
    "\n",
    "    # Passar pelo modelo\n",
    "    output = model(data)\n",
    "    print(f\"Model output shape: {output.shape}\")\n",
    "\n",
    "    # Ajustar para CrossEntropyLoss\n",
    "    output = output.view(-1, ntokens)  # Redimensionar para (seq_len * batch_size, ntokens)\n",
    "    target = target.view(-1)  # Redimensionar para (seq_len * batch_size)\n",
    "    print(f\"Adjusted output shape: {output.shape}\")\n",
    "    print(f\"Adjusted target shape: {target.shape}\")\n",
    "\n",
    "    # Calcular a perda\n",
    "    loss = criterion(output, target)\n",
    "    print(f\"Loss: {loss.item()}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1656cd27",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:08.012898Z",
     "iopub.status.busy": "2024-10-03T15:51:08.012133Z",
     "iopub.status.idle": "2024-10-03T15:51:08.336868Z",
     "shell.execute_reply": "2024-10-03T15:51:08.335383Z"
    },
    "papermill": {
     "duration": 0.428932,
     "end_time": "2024-10-03T15:51:08.340450",
     "exception": false,
     "start_time": "2024-10-03T15:51:07.911518",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data shape: torch.Size([35, 20])\n",
      "Target shape: torch.Size([35, 20])\n",
      "Model output shape: torch.Size([35, 20, 28378])\n",
      "Adjusted output shape: torch.Size([700, 28378])\n",
      "Adjusted target shape: torch.Size([700])\n",
      "Loss: 10.303860664367676\n"
     ]
    }
   ],
   "source": [
    "# Configurações de treinamento\n",
    "bptt = 35  # Exemplo de comprimento do batch de sequência\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# Definir o dispositivo (GPU ou CPU)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Pegue uma amostra para checar\n",
    "src = train_data[:batch_size]\n",
    "data, target = get_batch(src, 0, bptt, device)  # Exemplo de dados e alvos\n",
    "\n",
    "# Chamar a função com o dispositivo correto\n",
    "check_and_adjust_batch(data, target, model, criterion, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f5edc554",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:08.525865Z",
     "iopub.status.busy": "2024-10-03T15:51:08.525234Z",
     "iopub.status.idle": "2024-10-03T15:51:08.898479Z",
     "shell.execute_reply": "2024-10-03T15:51:08.897305Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.460666,
     "end_time": "2024-10-03T15:51:08.901659",
     "exception": false,
     "start_time": "2024-10-03T15:51:08.440993",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Inicializar o modelo\n",
    "model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "26ae41f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:09.069851Z",
     "iopub.status.busy": "2024-10-03T15:51:09.069264Z",
     "iopub.status.idle": "2024-10-03T15:51:09.084104Z",
     "shell.execute_reply": "2024-10-03T15:51:09.082756Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.102888,
     "end_time": "2024-10-03T15:51:09.087263",
     "exception": false,
     "start_time": "2024-10-03T15:51:08.984375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função de treinamento\n",
    "def train():\n",
    "    model.train()  # Ativar o modo de treinamento\n",
    "    total_loss = 0.\n",
    "    start_time = time.time()\n",
    "    for batch, i in enumerate(range(0, train_data.size(0) - 1, bptt)):\n",
    "        data, targets = get_batch(train_data, i, bptt,device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        output_flat = output.view(-1, ntokens)\n",
    "        loss = criterion(output_flat, targets.view(-1))\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
    "        optimizer.step()\n",
    "        total_loss += loss.item()\n",
    "        if batch % log_interval == 0 and batch > 0:\n",
    "            cur_loss = total_loss / log_interval\n",
    "            elapsed = time.time() - start_time\n",
    "            print('| epoch {:3d} | {:5d}/{:5d} batches | '\n",
    "                  'lr {:.5f} | loss {:5.2f} | ppl {:8.2f}'.format(\n",
    "                epoch, batch, len(train_data) // bptt,\n",
    "                lr, cur_loss, math.exp(cur_loss)))\n",
    "            total_loss = 0.\n",
    "            start_time = time.time()  # Reiniciar o tempo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "50fd2555",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:09.277677Z",
     "iopub.status.busy": "2024-10-03T15:51:09.276533Z",
     "iopub.status.idle": "2024-10-03T15:51:09.286351Z",
     "shell.execute_reply": "2024-10-03T15:51:09.285149Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.101485,
     "end_time": "2024-10-03T15:51:09.288897",
     "exception": false,
     "start_time": "2024-10-03T15:51:09.187412",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função de avaliação\n",
    "def evaluate(eval_model, data_source):\n",
    "    eval_model.eval()  # Ativar o modo de avaliação\n",
    "    total_loss = 0.\n",
    "    with torch.no_grad():\n",
    "        for i in range(0, data_source.size(0) - 1, bptt):\n",
    "            data, targets = get_batch(data_source, i, bptt,device)\n",
    "            output = eval_model(data)\n",
    "            output_flat = output.view(-1, ntokens)\n",
    "            total_loss += len(data) * criterion(output_flat, targets.view(-1)).item()\n",
    "    return total_loss / (len(data_source) - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "66911f1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:09.452375Z",
     "iopub.status.busy": "2024-10-03T15:51:09.451542Z",
     "iopub.status.idle": "2024-10-03T15:51:09.457743Z",
     "shell.execute_reply": "2024-10-03T15:51:09.456658Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.091516,
     "end_time": "2024-10-03T15:51:09.460670",
     "exception": false,
     "start_time": "2024-10-03T15:51:09.369154",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Configurações de treinamento\n",
    "n_epochs = 5\n",
    "lr = 5.0\n",
    "batch_size = 20\n",
    "bptt = 35\n",
    "log_interval = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b13621c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:09.630330Z",
     "iopub.status.busy": "2024-10-03T15:51:09.629102Z",
     "iopub.status.idle": "2024-10-03T15:51:09.636055Z",
     "shell.execute_reply": "2024-10-03T15:51:09.634879Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.094264,
     "end_time": "2024-10-03T15:51:09.638639",
     "exception": false,
     "start_time": "2024-10-03T15:51:09.544375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Inicializar o otimizador\n",
    "optimizer = optim.SGD(model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6828df64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:09.803119Z",
     "iopub.status.busy": "2024-10-03T15:51:09.801908Z",
     "iopub.status.idle": "2024-10-03T15:51:09.808014Z",
     "shell.execute_reply": "2024-10-03T15:51:09.806902Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.091152,
     "end_time": "2024-10-03T15:51:09.810544",
     "exception": false,
     "start_time": "2024-10-03T15:51:09.719392",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Inicializar o scheduler (opcional)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b0c302ea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:09.981206Z",
     "iopub.status.busy": "2024-10-03T15:51:09.980179Z",
     "iopub.status.idle": "2024-10-03T15:51:09.986454Z",
     "shell.execute_reply": "2024-10-03T15:51:09.985229Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 0.09577,
     "end_time": "2024-10-03T15:51:09.989101",
     "exception": false,
     "start_time": "2024-10-03T15:51:09.893331",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Treinamento e avaliação\n",
    "start_time = time.time()  # Iniciar a contagem do tempo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "72ecd43d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T15:51:10.159768Z",
     "iopub.status.busy": "2024-10-03T15:51:10.159218Z",
     "iopub.status.idle": "2024-10-03T16:25:05.375405Z",
     "shell.execute_reply": "2024-10-03T16:25:05.374143Z"
    },
    "executionInfo": {
     "elapsed": 1149648,
     "status": "error",
     "timestamp": 1726540919494,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 300
    },
    "id": "nALirb_dtDCV",
    "outputId": "4087d5db-4732-47d9-eea3-906fa73c1e8b",
    "papermill": {
     "duration": 2035.47452,
     "end_time": "2024-10-03T16:25:05.546679",
     "exception": false,
     "start_time": "2024-10-03T15:51:10.072159",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |   200/36490 batches | lr 5.00000 | loss  2.90 | ppl    18.17\n",
      "| epoch   1 |   400/36490 batches | lr 5.00000 | loss  2.10 | ppl     8.20\n",
      "| epoch   1 |   600/36490 batches | lr 5.00000 | loss  1.94 | ppl     6.94\n",
      "| epoch   1 |   800/36490 batches | lr 5.00000 | loss  1.69 | ppl     5.40\n",
      "| epoch   1 |  1000/36490 batches | lr 5.00000 | loss  1.60 | ppl     4.95\n",
      "| epoch   1 |  1200/36490 batches | lr 5.00000 | loss  1.67 | ppl     5.32\n",
      "| epoch   1 |  1400/36490 batches | lr 5.00000 | loss  1.46 | ppl     4.32\n",
      "| epoch   1 |  1600/36490 batches | lr 5.00000 | loss  1.37 | ppl     3.95\n",
      "| epoch   1 |  1800/36490 batches | lr 5.00000 | loss  1.42 | ppl     4.13\n",
      "| epoch   1 |  2000/36490 batches | lr 5.00000 | loss  1.46 | ppl     4.29\n",
      "| epoch   1 |  2200/36490 batches | lr 5.00000 | loss  1.32 | ppl     3.76\n",
      "| epoch   1 |  2400/36490 batches | lr 5.00000 | loss  1.24 | ppl     3.44\n",
      "| epoch   1 |  2600/36490 batches | lr 5.00000 | loss  1.40 | ppl     4.05\n",
      "| epoch   1 |  2800/36490 batches | lr 5.00000 | loss  1.18 | ppl     3.24\n",
      "| epoch   1 |  3000/36490 batches | lr 5.00000 | loss  1.11 | ppl     3.03\n",
      "| epoch   1 |  3200/36490 batches | lr 5.00000 | loss  1.20 | ppl     3.31\n",
      "| epoch   1 |  3400/36490 batches | lr 5.00000 | loss  1.03 | ppl     2.80\n",
      "| epoch   1 |  3600/36490 batches | lr 5.00000 | loss  1.28 | ppl     3.59\n",
      "| epoch   1 |  3800/36490 batches | lr 5.00000 | loss  1.61 | ppl     5.01\n",
      "| epoch   1 |  4000/36490 batches | lr 5.00000 | loss  1.37 | ppl     3.93\n",
      "| epoch   1 |  4200/36490 batches | lr 5.00000 | loss  1.09 | ppl     2.98\n",
      "| epoch   1 |  4400/36490 batches | lr 5.00000 | loss  1.42 | ppl     4.14\n",
      "| epoch   1 |  4600/36490 batches | lr 5.00000 | loss  1.23 | ppl     3.43\n",
      "| epoch   1 |  4800/36490 batches | lr 5.00000 | loss  1.46 | ppl     4.31\n",
      "| epoch   1 |  5000/36490 batches | lr 5.00000 | loss  1.32 | ppl     3.74\n",
      "| epoch   1 |  5200/36490 batches | lr 5.00000 | loss  0.99 | ppl     2.69\n",
      "| epoch   1 |  5400/36490 batches | lr 5.00000 | loss  1.25 | ppl     3.48\n",
      "| epoch   1 |  5600/36490 batches | lr 5.00000 | loss  0.99 | ppl     2.70\n",
      "| epoch   1 |  5800/36490 batches | lr 5.00000 | loss  1.35 | ppl     3.87\n",
      "| epoch   1 |  6000/36490 batches | lr 5.00000 | loss  1.07 | ppl     2.91\n",
      "| epoch   1 |  6200/36490 batches | lr 5.00000 | loss  1.35 | ppl     3.87\n",
      "| epoch   1 |  6400/36490 batches | lr 5.00000 | loss  1.15 | ppl     3.16\n",
      "| epoch   1 |  6600/36490 batches | lr 5.00000 | loss  1.18 | ppl     3.24\n",
      "| epoch   1 |  6800/36490 batches | lr 5.00000 | loss  1.41 | ppl     4.10\n",
      "| epoch   1 |  7000/36490 batches | lr 5.00000 | loss  1.12 | ppl     3.06\n",
      "| epoch   1 |  7200/36490 batches | lr 5.00000 | loss  1.06 | ppl     2.88\n",
      "| epoch   1 |  7400/36490 batches | lr 5.00000 | loss  1.14 | ppl     3.11\n",
      "| epoch   1 |  7600/36490 batches | lr 5.00000 | loss  1.23 | ppl     3.41\n",
      "| epoch   1 |  7800/36490 batches | lr 5.00000 | loss  1.06 | ppl     2.89\n",
      "| epoch   1 |  8000/36490 batches | lr 5.00000 | loss  1.22 | ppl     3.39\n",
      "| epoch   1 |  8200/36490 batches | lr 5.00000 | loss  0.89 | ppl     2.43\n",
      "| epoch   1 |  8400/36490 batches | lr 5.00000 | loss  0.93 | ppl     2.54\n",
      "| epoch   1 |  8600/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.09\n",
      "| epoch   1 |  8800/36490 batches | lr 5.00000 | loss  0.87 | ppl     2.38\n",
      "| epoch   1 |  9000/36490 batches | lr 5.00000 | loss  0.97 | ppl     2.64\n",
      "| epoch   1 |  9200/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.26\n",
      "| epoch   1 |  9400/36490 batches | lr 5.00000 | loss  0.94 | ppl     2.56\n",
      "| epoch   1 |  9600/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.26\n",
      "| epoch   1 |  9800/36490 batches | lr 5.00000 | loss  0.89 | ppl     2.44\n",
      "| epoch   1 | 10000/36490 batches | lr 5.00000 | loss  0.92 | ppl     2.52\n",
      "| epoch   1 | 10200/36490 batches | lr 5.00000 | loss  0.87 | ppl     2.38\n",
      "| epoch   1 | 10400/36490 batches | lr 5.00000 | loss  1.01 | ppl     2.75\n",
      "| epoch   1 | 10600/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.10\n",
      "| epoch   1 | 10800/36490 batches | lr 5.00000 | loss  1.02 | ppl     2.78\n",
      "| epoch   1 | 11000/36490 batches | lr 5.00000 | loss  1.01 | ppl     2.76\n",
      "| epoch   1 | 11200/36490 batches | lr 5.00000 | loss  0.88 | ppl     2.42\n",
      "| epoch   1 | 11400/36490 batches | lr 5.00000 | loss  1.01 | ppl     2.75\n",
      "| epoch   1 | 11600/36490 batches | lr 5.00000 | loss  1.33 | ppl     3.79\n",
      "| epoch   1 | 11800/36490 batches | lr 5.00000 | loss  1.11 | ppl     3.04\n",
      "| epoch   1 | 12000/36490 batches | lr 5.00000 | loss  1.03 | ppl     2.80\n",
      "| epoch   1 | 12200/36490 batches | lr 5.00000 | loss  0.91 | ppl     2.49\n",
      "| epoch   1 | 12400/36490 batches | lr 5.00000 | loss  1.16 | ppl     3.18\n",
      "| epoch   1 | 12600/36490 batches | lr 5.00000 | loss  1.05 | ppl     2.86\n",
      "| epoch   1 | 12800/36490 batches | lr 5.00000 | loss  1.27 | ppl     3.55\n",
      "| epoch   1 | 13000/36490 batches | lr 5.00000 | loss  1.07 | ppl     2.92\n",
      "| epoch   1 | 13200/36490 batches | lr 5.00000 | loss  1.26 | ppl     3.54\n",
      "| epoch   1 | 13400/36490 batches | lr 5.00000 | loss  1.53 | ppl     4.60\n",
      "| epoch   1 | 13600/36490 batches | lr 5.00000 | loss  1.05 | ppl     2.87\n",
      "| epoch   1 | 13800/36490 batches | lr 5.00000 | loss  0.82 | ppl     2.28\n",
      "| epoch   1 | 14000/36490 batches | lr 5.00000 | loss  0.98 | ppl     2.66\n",
      "| epoch   1 | 14200/36490 batches | lr 5.00000 | loss  0.95 | ppl     2.59\n",
      "| epoch   1 | 14400/36490 batches | lr 5.00000 | loss  1.22 | ppl     3.39\n",
      "| epoch   1 | 14600/36490 batches | lr 5.00000 | loss  1.13 | ppl     3.08\n",
      "| epoch   1 | 14800/36490 batches | lr 5.00000 | loss  1.49 | ppl     4.43\n",
      "| epoch   1 | 15000/36490 batches | lr 5.00000 | loss  1.15 | ppl     3.15\n",
      "| epoch   1 | 15200/36490 batches | lr 5.00000 | loss  1.28 | ppl     3.59\n",
      "| epoch   1 | 15400/36490 batches | lr 5.00000 | loss  0.85 | ppl     2.34\n",
      "| epoch   1 | 15600/36490 batches | lr 5.00000 | loss  1.11 | ppl     3.03\n",
      "| epoch   1 | 15800/36490 batches | lr 5.00000 | loss  0.88 | ppl     2.41\n",
      "| epoch   1 | 16000/36490 batches | lr 5.00000 | loss  1.15 | ppl     3.16\n",
      "| epoch   1 | 16200/36490 batches | lr 5.00000 | loss  0.94 | ppl     2.56\n",
      "| epoch   1 | 16400/36490 batches | lr 5.00000 | loss  0.83 | ppl     2.29\n",
      "| epoch   1 | 16600/36490 batches | lr 5.00000 | loss  0.80 | ppl     2.23\n",
      "| epoch   1 | 16800/36490 batches | lr 5.00000 | loss  0.87 | ppl     2.38\n",
      "| epoch   1 | 17000/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.25\n",
      "| epoch   1 | 17200/36490 batches | lr 5.00000 | loss  0.85 | ppl     2.34\n",
      "| epoch   1 | 17400/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.24\n",
      "| epoch   1 | 17600/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   1 | 17800/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.26\n",
      "| epoch   1 | 18000/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.16\n",
      "| epoch   1 | 18200/36490 batches | lr 5.00000 | loss  0.89 | ppl     2.44\n",
      "| epoch   1 | 18400/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   1 | 18600/36490 batches | lr 5.00000 | loss  1.02 | ppl     2.76\n",
      "| epoch   1 | 18800/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.24\n",
      "| epoch   1 | 19000/36490 batches | lr 5.00000 | loss  0.90 | ppl     2.45\n",
      "| epoch   1 | 19200/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.15\n",
      "| epoch   1 | 19400/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   1 | 19600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   1 | 19800/36490 batches | lr 5.00000 | loss  0.90 | ppl     2.46\n",
      "| epoch   1 | 20000/36490 batches | lr 5.00000 | loss  0.92 | ppl     2.50\n",
      "| epoch   1 | 20200/36490 batches | lr 5.00000 | loss  0.80 | ppl     2.24\n",
      "| epoch   1 | 20400/36490 batches | lr 5.00000 | loss  0.84 | ppl     2.32\n",
      "| epoch   1 | 20600/36490 batches | lr 5.00000 | loss  0.85 | ppl     2.34\n",
      "| epoch   1 | 20800/36490 batches | lr 5.00000 | loss  0.87 | ppl     2.39\n",
      "| epoch   1 | 21000/36490 batches | lr 5.00000 | loss  0.82 | ppl     2.28\n",
      "| epoch   1 | 21200/36490 batches | lr 5.00000 | loss  0.79 | ppl     2.21\n",
      "| epoch   1 | 21400/36490 batches | lr 5.00000 | loss  0.84 | ppl     2.32\n",
      "| epoch   1 | 21600/36490 batches | lr 5.00000 | loss  0.94 | ppl     2.55\n",
      "| epoch   1 | 21800/36490 batches | lr 5.00000 | loss  0.79 | ppl     2.20\n",
      "| epoch   1 | 22000/36490 batches | lr 5.00000 | loss  0.84 | ppl     2.32\n",
      "| epoch   1 | 22200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   1 | 22400/36490 batches | lr 5.00000 | loss  0.90 | ppl     2.46\n",
      "| epoch   1 | 22600/36490 batches | lr 5.00000 | loss  1.17 | ppl     3.24\n",
      "| epoch   1 | 22800/36490 batches | lr 5.00000 | loss  0.96 | ppl     2.61\n",
      "| epoch   1 | 23000/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.25\n",
      "| epoch   1 | 23200/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   1 | 23400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   1 | 23600/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   1 | 23800/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   1 | 24000/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   1 | 24200/36490 batches | lr 5.00000 | loss  0.86 | ppl     2.36\n",
      "| epoch   1 | 24400/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.81\n",
      "| epoch   1 | 24600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   1 | 24800/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   1 | 25000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.94\n",
      "| epoch   1 | 25200/36490 batches | lr 5.00000 | loss  0.78 | ppl     2.18\n",
      "| epoch   1 | 25400/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   1 | 25600/36490 batches | lr 5.00000 | loss  0.83 | ppl     2.29\n",
      "| epoch   1 | 25800/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.25\n",
      "| epoch   1 | 26000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   1 | 26200/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   1 | 26400/36490 batches | lr 5.00000 | loss  0.75 | ppl     2.12\n",
      "| epoch   1 | 26600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   1 | 26800/36490 batches | lr 5.00000 | loss  0.78 | ppl     2.19\n",
      "| epoch   1 | 27000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   1 | 27200/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   1 | 27400/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   1 | 27600/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   1 | 27800/36490 batches | lr 5.00000 | loss  0.75 | ppl     2.13\n",
      "| epoch   1 | 28000/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.00\n",
      "| epoch   1 | 28200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   1 | 28400/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   1 | 28600/36490 batches | lr 5.00000 | loss  0.80 | ppl     2.24\n",
      "| epoch   1 | 28800/36490 batches | lr 5.00000 | loss  0.78 | ppl     2.19\n",
      "| epoch   1 | 29000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   1 | 29200/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   1 | 29400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   1 | 29600/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   1 | 29800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   1 | 30000/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.98\n",
      "| epoch   1 | 30200/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   1 | 30400/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   1 | 30600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   1 | 30800/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   1 | 31000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   1 | 31200/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   1 | 31400/36490 batches | lr 5.00000 | loss  0.75 | ppl     2.12\n",
      "| epoch   1 | 31600/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   1 | 31800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   1 | 32000/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.02\n",
      "| epoch   1 | 32200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   1 | 32400/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   1 | 32600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   1 | 32800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   1 | 33000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   1 | 33200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   1 | 33400/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   1 | 33600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   1 | 33800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   1 | 34000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   1 | 34200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   1 | 34400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   1 | 34600/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.02\n",
      "| epoch   1 | 34800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   1 | 35000/36490 batches | lr 5.00000 | loss  0.82 | ppl     2.27\n",
      "| epoch   1 | 35200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   1 | 35400/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   1 | 35600/36490 batches | lr 5.00000 | loss  0.69 | ppl     2.00\n",
      "| epoch   1 | 35800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   1 | 36000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   1 | 36200/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.16\n",
      "| epoch   1 | 36400/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   1 | time: 404.27s | valid loss  1.03 | valid ppl     2.81\n",
      "-----------------------------------------------------------------------------------------\n",
      "| epoch   2 |   200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   2 |   400/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.14\n",
      "| epoch   2 |   600/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.14\n",
      "| epoch   2 |   800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   2 |  1000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   2 |  1200/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   2 |  1400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   2 |  1600/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 |  1800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   2 |  2000/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.13\n",
      "| epoch   2 |  2200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   2 |  2400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   2 |  2600/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   2 |  2800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   2 |  3000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   2 |  3200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   2 |  3400/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   2 |  3600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   2 |  3800/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   2 |  4000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   2 |  4200/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.66\n",
      "| epoch   2 |  4400/36490 batches | lr 5.00000 | loss  0.92 | ppl     2.52\n",
      "| epoch   2 |  4600/36490 batches | lr 5.00000 | loss  1.31 | ppl     3.71\n",
      "| epoch   2 |  4800/36490 batches | lr 5.00000 | loss  1.29 | ppl     3.62\n",
      "| epoch   2 |  5000/36490 batches | lr 5.00000 | loss  1.15 | ppl     3.16\n",
      "| epoch   2 |  5200/36490 batches | lr 5.00000 | loss  0.82 | ppl     2.28\n",
      "| epoch   2 |  5400/36490 batches | lr 5.00000 | loss  1.02 | ppl     2.77\n",
      "| epoch   2 |  5600/36490 batches | lr 5.00000 | loss  0.88 | ppl     2.40\n",
      "| epoch   2 |  5800/36490 batches | lr 5.00000 | loss  1.14 | ppl     3.14\n",
      "| epoch   2 |  6000/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.10\n",
      "| epoch   2 |  6200/36490 batches | lr 5.00000 | loss  1.00 | ppl     2.71\n",
      "| epoch   2 |  6400/36490 batches | lr 5.00000 | loss  0.82 | ppl     2.27\n",
      "| epoch   2 |  6600/36490 batches | lr 5.00000 | loss  0.78 | ppl     2.19\n",
      "| epoch   2 |  6800/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.15\n",
      "| epoch   2 |  7000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 |  7200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   2 |  7400/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   2 |  7600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   2 |  7800/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   2 |  8000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   2 |  8200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   2 |  8400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   2 |  8600/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.66\n",
      "| epoch   2 |  8800/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   2 |  9000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 |  9200/36490 batches | lr 5.00000 | loss  0.46 | ppl     1.58\n",
      "| epoch   2 |  9400/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.81\n",
      "| epoch   2 |  9600/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.66\n",
      "| epoch   2 |  9800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 | 10000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   2 | 10200/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   2 | 10400/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 | 10600/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.66\n",
      "| epoch   2 | 10800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   2 | 11000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   2 | 11200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   2 | 11400/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   2 | 11600/36490 batches | lr 5.00000 | loss  0.79 | ppl     2.21\n",
      "| epoch   2 | 11800/36490 batches | lr 5.00000 | loss  1.10 | ppl     3.00\n",
      "| epoch   2 | 12000/36490 batches | lr 5.00000 | loss  1.12 | ppl     3.06\n",
      "| epoch   2 | 12200/36490 batches | lr 5.00000 | loss  1.02 | ppl     2.78\n",
      "| epoch   2 | 12400/36490 batches | lr 5.00000 | loss  1.27 | ppl     3.55\n",
      "| epoch   2 | 12600/36490 batches | lr 5.00000 | loss  1.00 | ppl     2.73\n",
      "| epoch   2 | 12800/36490 batches | lr 5.00000 | loss  1.22 | ppl     3.40\n",
      "| epoch   2 | 13000/36490 batches | lr 5.00000 | loss  1.24 | ppl     3.45\n",
      "| epoch   2 | 13200/36490 batches | lr 5.00000 | loss  1.19 | ppl     3.29\n",
      "| epoch   2 | 13400/36490 batches | lr 5.00000 | loss  1.16 | ppl     3.19\n",
      "| epoch   2 | 13600/36490 batches | lr 5.00000 | loss  1.06 | ppl     2.89\n",
      "| epoch   2 | 13800/36490 batches | lr 5.00000 | loss  0.99 | ppl     2.69\n",
      "| epoch   2 | 14000/36490 batches | lr 5.00000 | loss  1.10 | ppl     2.99\n",
      "| epoch   2 | 14200/36490 batches | lr 5.00000 | loss  1.07 | ppl     2.92\n",
      "| epoch   2 | 14400/36490 batches | lr 5.00000 | loss  1.08 | ppl     2.95\n",
      "| epoch   2 | 14600/36490 batches | lr 5.00000 | loss  1.17 | ppl     3.23\n",
      "| epoch   2 | 14800/36490 batches | lr 5.00000 | loss  1.18 | ppl     3.24\n",
      "| epoch   2 | 15000/36490 batches | lr 5.00000 | loss  0.99 | ppl     2.68\n",
      "| epoch   2 | 15200/36490 batches | lr 5.00000 | loss  1.05 | ppl     2.86\n",
      "| epoch   2 | 15400/36490 batches | lr 5.00000 | loss  0.85 | ppl     2.33\n",
      "| epoch   2 | 15600/36490 batches | lr 5.00000 | loss  1.13 | ppl     3.09\n",
      "| epoch   2 | 15800/36490 batches | lr 5.00000 | loss  0.88 | ppl     2.41\n",
      "| epoch   2 | 16000/36490 batches | lr 5.00000 | loss  1.00 | ppl     2.72\n",
      "| epoch   2 | 16200/36490 batches | lr 5.00000 | loss  0.94 | ppl     2.56\n",
      "| epoch   2 | 16400/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.25\n",
      "| epoch   2 | 16600/36490 batches | lr 5.00000 | loss  0.86 | ppl     2.36\n",
      "| epoch   2 | 16800/36490 batches | lr 5.00000 | loss  0.87 | ppl     2.39\n",
      "| epoch   2 | 17000/36490 batches | lr 5.00000 | loss  0.87 | ppl     2.38\n",
      "| epoch   2 | 17200/36490 batches | lr 5.00000 | loss  0.94 | ppl     2.56\n",
      "| epoch   2 | 17400/36490 batches | lr 5.00000 | loss  0.81 | ppl     2.26\n",
      "| epoch   2 | 17600/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   2 | 17800/36490 batches | lr 5.00000 | loss  0.83 | ppl     2.28\n",
      "| epoch   2 | 18000/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   2 | 18200/36490 batches | lr 5.00000 | loss  1.16 | ppl     3.19\n",
      "| epoch   2 | 18400/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.98\n",
      "| epoch   2 | 18600/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.10\n",
      "| epoch   2 | 18800/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 | 19000/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   2 | 19200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   2 | 19400/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   2 | 19600/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   2 | 19800/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   2 | 20000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   2 | 20200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   2 | 20400/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   2 | 20600/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   2 | 20800/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   2 | 21000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   2 | 21200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   2 | 21400/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   2 | 21600/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   2 | 21800/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   2 | 22000/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   2 | 22200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.72\n",
      "| epoch   2 | 22400/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   2 | 22600/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   2 | 22800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   2 | 23000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   2 | 23200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   2 | 23400/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 | 23600/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   2 | 23800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   2 | 24000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   2 | 24200/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.14\n",
      "| epoch   2 | 24400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   2 | 24600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   2 | 24800/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   2 | 25000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 | 25200/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.09\n",
      "| epoch   2 | 25400/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   2 | 25600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   2 | 25800/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   2 | 26000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   2 | 26200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   2 | 26400/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   2 | 26600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   2 | 26800/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   2 | 27000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   2 | 27200/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 | 27400/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 | 27600/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 | 27800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   2 | 28000/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   2 | 28200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   2 | 28400/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   2 | 28600/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   2 | 28800/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   2 | 29000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   2 | 29200/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   2 | 29400/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   2 | 29600/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   2 | 29800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   2 | 30000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   2 | 30200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   2 | 30400/36490 batches | lr 5.00000 | loss  0.46 | ppl     1.59\n",
      "| epoch   2 | 30600/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   2 | 30800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   2 | 31000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   2 | 31200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   2 | 31400/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   2 | 31600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   2 | 31800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   2 | 32000/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   2 | 32200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   2 | 32400/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 | 32600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   2 | 32800/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   2 | 33000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   2 | 33200/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   2 | 33400/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   2 | 33600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   2 | 33800/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   2 | 34000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   2 | 34200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.81\n",
      "| epoch   2 | 34400/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   2 | 34600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   2 | 34800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   2 | 35000/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.13\n",
      "| epoch   2 | 35200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   2 | 35400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.89\n",
      "| epoch   2 | 35600/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   2 | 35800/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   2 | 36000/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   2 | 36200/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.09\n",
      "| epoch   2 | 36400/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   2 | time: 807.72s | valid loss  6.04 | valid ppl   420.48\n",
      "-----------------------------------------------------------------------------------------\n",
      "| epoch   3 |   200/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 |   400/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   3 |   600/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   3 |   800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   3 |  1000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   3 |  1200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   3 |  1400/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   3 |  1600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 |  1800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   3 |  2000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   3 |  2200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 |  2400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   3 |  2600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 |  2800/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   3 |  3000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   3 |  3200/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.66\n",
      "| epoch   3 |  3400/36490 batches | lr 5.00000 | loss  0.46 | ppl     1.59\n",
      "| epoch   3 |  3600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   3 |  3800/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.02\n",
      "| epoch   3 |  4000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 |  4200/36490 batches | lr 5.00000 | loss  0.46 | ppl     1.59\n",
      "| epoch   3 |  4400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   3 |  4600/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 |  4800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   3 |  5000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   3 |  5200/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   3 |  5400/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   3 |  5600/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.64\n",
      "| epoch   3 |  5800/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.02\n",
      "| epoch   3 |  6000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   3 |  6200/36490 batches | lr 5.00000 | loss  1.10 | ppl     2.99\n",
      "| epoch   3 |  6400/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.16\n",
      "| epoch   3 |  6600/36490 batches | lr 5.00000 | loss  0.98 | ppl     2.68\n",
      "| epoch   3 |  6800/36490 batches | lr 5.00000 | loss  0.89 | ppl     2.43\n",
      "| epoch   3 |  7000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   3 |  7200/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   3 |  7400/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 |  7600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 |  7800/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   3 |  8000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   3 |  8200/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.65\n",
      "| epoch   3 |  8400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   3 |  8600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   3 |  8800/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   3 |  9000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   3 |  9200/36490 batches | lr 5.00000 | loss  0.43 | ppl     1.54\n",
      "| epoch   3 |  9400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   3 |  9600/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.64\n",
      "| epoch   3 |  9800/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   3 | 10000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 | 10200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 | 10400/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   3 | 10600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   3 | 10800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 | 11000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 | 11200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   3 | 11400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.89\n",
      "| epoch   3 | 11600/36490 batches | lr 5.00000 | loss  0.91 | ppl     2.47\n",
      "| epoch   3 | 11800/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.09\n",
      "| epoch   3 | 12000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.89\n",
      "| epoch   3 | 12200/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   3 | 12400/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   3 | 12600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   3 | 12800/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   3 | 13000/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   3 | 13200/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 | 13400/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   3 | 13600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   3 | 13800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   3 | 14000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 | 14200/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 | 14400/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   3 | 14600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   3 | 14800/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.02\n",
      "| epoch   3 | 15000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   3 | 15200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   3 | 15400/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 | 15600/36490 batches | lr 5.00000 | loss  0.69 | ppl     2.00\n",
      "| epoch   3 | 15800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.74\n",
      "| epoch   3 | 16000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 | 16200/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 | 16400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   3 | 16600/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   3 | 16800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.74\n",
      "| epoch   3 | 17000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   3 | 17200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   3 | 17400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   3 | 17600/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.60\n",
      "| epoch   3 | 17800/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.64\n",
      "| epoch   3 | 18000/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.66\n",
      "| epoch   3 | 18200/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   3 | 18400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   3 | 18600/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   3 | 18800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   3 | 19000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 19200/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   3 | 19400/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.65\n",
      "| epoch   3 | 19600/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.65\n",
      "| epoch   3 | 19800/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   3 | 20000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   3 | 20200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   3 | 20400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 | 20600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   3 | 20800/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.10\n",
      "| epoch   3 | 21000/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   3 | 21200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   3 | 21400/36490 batches | lr 5.00000 | loss  0.69 | ppl     2.00\n",
      "| epoch   3 | 21600/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   3 | 21800/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   3 | 22000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 22200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   3 | 22400/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   3 | 22600/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   3 | 22800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   3 | 23000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   3 | 23200/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   3 | 23400/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 | 23600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   3 | 23800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   3 | 24000/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.02\n",
      "| epoch   3 | 24200/36490 batches | lr 5.00000 | loss  1.00 | ppl     2.72\n",
      "| epoch   3 | 24400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 | 24600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 | 24800/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   3 | 25000/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.98\n",
      "| epoch   3 | 25200/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.16\n",
      "| epoch   3 | 25400/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.17\n",
      "| epoch   3 | 25600/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.09\n",
      "| epoch   3 | 25800/36490 batches | lr 5.00000 | loss  0.88 | ppl     2.40\n",
      "| epoch   3 | 26000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   3 | 26200/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   3 | 26400/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   3 | 26600/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   3 | 26800/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   3 | 27000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   3 | 27200/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   3 | 27400/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 | 27600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.98\n",
      "| epoch   3 | 27800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 | 28000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   3 | 28200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 | 28400/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   3 | 28600/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   3 | 28800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 29000/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   3 | 29200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   3 | 29400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   3 | 29600/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   3 | 29800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   3 | 30000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 30200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 | 30400/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   3 | 30600/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   3 | 30800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.89\n",
      "| epoch   3 | 31000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   3 | 31200/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   3 | 31400/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.11\n",
      "| epoch   3 | 31600/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 31800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   3 | 32000/36490 batches | lr 5.00000 | loss  0.69 | ppl     2.00\n",
      "| epoch   3 | 32200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   3 | 32400/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   3 | 32600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   3 | 32800/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   3 | 33000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   3 | 33200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   3 | 33400/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   3 | 33600/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 33800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   3 | 34000/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.74\n",
      "| epoch   3 | 34200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   3 | 34400/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   3 | 34600/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   3 | 34800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   3 | 35000/36490 batches | lr 5.00000 | loss  0.78 | ppl     2.19\n",
      "| epoch   3 | 35200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   3 | 35400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   3 | 35600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   3 | 35800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   3 | 36000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   3 | 36200/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.15\n",
      "| epoch   3 | 36400/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   3 | time: 1208.34s | valid loss  2.45 | valid ppl    11.56\n",
      "-----------------------------------------------------------------------------------------\n",
      "| epoch   4 |   200/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   4 |   400/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   4 |   600/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.09\n",
      "| epoch   4 |   800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 |  1000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   4 |  1200/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   4 |  1400/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   4 |  1600/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.14\n",
      "| epoch   4 |  1800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 |  2000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   4 |  2200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   4 |  2400/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 |  2600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   4 |  2800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 |  3000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   4 |  3200/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   4 |  3400/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   4 |  3600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   4 |  3800/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   4 |  4000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   4 |  4200/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   4 |  4400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   4 |  4600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   4 |  4800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   4 |  5000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   4 |  5200/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   4 |  5400/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   4 |  5600/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.64\n",
      "| epoch   4 |  5800/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   4 |  6000/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.67\n",
      "| epoch   4 |  6200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   4 |  6400/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 |  6600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   4 |  6800/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   4 |  7000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 |  7200/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.62\n",
      "| epoch   4 |  7400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.74\n",
      "| epoch   4 |  7600/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 |  7800/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.60\n",
      "| epoch   4 |  8000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 |  8200/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   4 |  8400/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 |  8600/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.61\n",
      "| epoch   4 |  8800/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 |  9000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   4 |  9200/36490 batches | lr 5.00000 | loss  0.43 | ppl     1.53\n",
      "| epoch   4 |  9400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   4 |  9600/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   4 |  9800/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 10000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   4 | 10200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.78\n",
      "| epoch   4 | 10400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.91\n",
      "| epoch   4 | 10600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   4 | 10800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   4 | 11000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   4 | 11200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   4 | 11400/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   4 | 11600/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.14\n",
      "| epoch   4 | 11800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   4 | 12000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   4 | 12200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 12400/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   4 | 12600/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   4 | 12800/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   4 | 13000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   4 | 13200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   4 | 13400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 13600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 13800/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 | 14000/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 14200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   4 | 14400/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   4 | 14600/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   4 | 14800/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 15000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 | 15200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   4 | 15400/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   4 | 15600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.98\n",
      "| epoch   4 | 15800/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   4 | 16000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   4 | 16200/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   4 | 16400/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 | 16600/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.67\n",
      "| epoch   4 | 16800/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   4 | 17000/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   4 | 17200/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 | 17400/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 | 17600/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.59\n",
      "| epoch   4 | 17800/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   4 | 18000/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.60\n",
      "| epoch   4 | 18200/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   4 | 18400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   4 | 18600/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   4 | 18800/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 19000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   4 | 19200/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 | 19400/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   4 | 19600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   4 | 19800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   4 | 20000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   4 | 20200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   4 | 20400/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   4 | 20600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 20800/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   4 | 21000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 21200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 | 21400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 21600/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   4 | 21800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   4 | 22000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   4 | 22200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   4 | 22400/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.96\n",
      "| epoch   4 | 22600/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   4 | 22800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   4 | 23000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   4 | 23200/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   4 | 23400/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 23600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   4 | 23800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 | 24000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   4 | 24200/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.10\n",
      "| epoch   4 | 24400/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.71\n",
      "| epoch   4 | 24600/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   4 | 24800/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   4 | 25000/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   4 | 25200/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   4 | 25400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   4 | 25600/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   4 | 25800/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   4 | 26000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 | 26200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   4 | 26400/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   4 | 26600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 26800/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   4 | 27000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 | 27200/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 27400/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   4 | 27600/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   4 | 27800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.81\n",
      "| epoch   4 | 28000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   4 | 28200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   4 | 28400/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.64\n",
      "| epoch   4 | 28600/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   4 | 28800/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.79\n",
      "| epoch   4 | 29000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   4 | 29200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 | 29400/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 | 29600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   4 | 29800/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 | 30000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   4 | 30200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   4 | 30400/36490 batches | lr 5.00000 | loss  0.45 | ppl     1.57\n",
      "| epoch   4 | 30600/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   4 | 30800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   4 | 31000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   4 | 31200/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   4 | 31400/36490 batches | lr 5.00000 | loss  0.69 | ppl     2.00\n",
      "| epoch   4 | 31600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   4 | 31800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.76\n",
      "| epoch   4 | 32000/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   4 | 32200/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   4 | 32400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   4 | 32600/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.74\n",
      "| epoch   4 | 32800/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 | 33000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   4 | 33200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 | 33400/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.98\n",
      "| epoch   4 | 33600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 | 33800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   4 | 34000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   4 | 34200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   4 | 34400/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   4 | 34600/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   4 | 34800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   4 | 35000/36490 batches | lr 5.00000 | loss  0.75 | ppl     2.12\n",
      "| epoch   4 | 35200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.81\n",
      "| epoch   4 | 35400/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.85\n",
      "| epoch   4 | 35600/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   4 | 35800/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   4 | 36000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   4 | 36200/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   4 | 36400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   4 | time: 1618.99s | valid loss 21.69 | valid ppl 2636897620.20\n",
      "-----------------------------------------------------------------------------------------\n",
      "| epoch   5 |   200/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.87\n",
      "| epoch   5 |   400/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   5 |   600/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.05\n",
      "| epoch   5 |   800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   5 |  1000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 |  1200/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   5 |  1400/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 |  1600/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   5 |  1800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 |  2000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   5 |  2200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.78\n",
      "| epoch   5 |  2400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   5 |  2600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 |  2800/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.66\n",
      "| epoch   5 |  3000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.71\n",
      "| epoch   5 |  3200/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   5 |  3400/36490 batches | lr 5.00000 | loss  0.46 | ppl     1.59\n",
      "| epoch   5 |  3600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   5 |  3800/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   5 |  4000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   5 |  4200/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   5 |  4400/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   5 |  4600/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   5 |  4800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 |  5000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 |  5200/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.61\n",
      "| epoch   5 |  5400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.74\n",
      "| epoch   5 |  5600/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   5 |  5800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 |  6000/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.67\n",
      "| epoch   5 |  6200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 |  6400/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.66\n",
      "| epoch   5 |  6600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 |  6800/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   5 |  7000/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.70\n",
      "| epoch   5 |  7200/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.63\n",
      "| epoch   5 |  7400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   5 |  7600/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   5 |  7800/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.60\n",
      "| epoch   5 |  8000/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   5 |  8200/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.65\n",
      "| epoch   5 |  8400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.67\n",
      "| epoch   5 |  8600/36490 batches | lr 5.00000 | loss  0.53 | ppl     1.69\n",
      "| epoch   5 |  8800/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   5 |  9000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 |  9200/36490 batches | lr 5.00000 | loss  0.43 | ppl     1.54\n",
      "| epoch   5 |  9400/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   5 |  9600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   5 |  9800/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   5 | 10000/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.08\n",
      "| epoch   5 | 10200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   5 | 10400/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   5 | 10600/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.64\n",
      "| epoch   5 | 10800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 11000/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   5 | 11200/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   5 | 11400/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.03\n",
      "| epoch   5 | 11600/36490 batches | lr 5.00000 | loss  0.85 | ppl     2.34\n",
      "| epoch   5 | 11800/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   5 | 12000/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   5 | 12200/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 | 12400/36490 batches | lr 5.00000 | loss  0.76 | ppl     2.14\n",
      "| epoch   5 | 12600/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 | 12800/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   5 | 13000/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   5 | 13200/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   5 | 13400/36490 batches | lr 5.00000 | loss  0.69 | ppl     2.00\n",
      "| epoch   5 | 13600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   5 | 13800/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   5 | 14000/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 14200/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.81\n",
      "| epoch   5 | 14400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   5 | 14600/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 | 14800/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   5 | 15000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   5 | 15200/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.98\n",
      "| epoch   5 | 15400/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 | 15600/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   5 | 15800/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   5 | 16000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   5 | 16200/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   5 | 16400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   5 | 16600/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   5 | 16800/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   5 | 17000/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   5 | 17200/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   5 | 17400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   5 | 17600/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.60\n",
      "| epoch   5 | 17800/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   5 | 18000/36490 batches | lr 5.00000 | loss  0.47 | ppl     1.61\n",
      "| epoch   5 | 18200/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   5 | 18400/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   5 | 18600/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   5 | 18800/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.92\n",
      "| epoch   5 | 19000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   5 | 19200/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   5 | 19400/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   5 | 19600/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.64\n",
      "| epoch   5 | 19800/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.94\n",
      "| epoch   5 | 20000/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 | 20200/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.90\n",
      "| epoch   5 | 20400/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.85\n",
      "| epoch   5 | 20600/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 | 20800/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   5 | 21000/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   5 | 21200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   5 | 21400/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   5 | 21600/36490 batches | lr 5.00000 | loss  0.77 | ppl     2.16\n",
      "| epoch   5 | 21800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 | 22000/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 | 22200/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   5 | 22400/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.98\n",
      "| epoch   5 | 22600/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.01\n",
      "| epoch   5 | 22800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 23000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   5 | 23200/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   5 | 23400/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 | 23600/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   5 | 23800/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 24000/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   5 | 24200/36490 batches | lr 5.00000 | loss  0.75 | ppl     2.13\n",
      "| epoch   5 | 24400/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.76\n",
      "| epoch   5 | 24600/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   5 | 24800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   5 | 25000/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 | 25200/36490 batches | lr 5.00000 | loss  0.73 | ppl     2.07\n",
      "| epoch   5 | 25400/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 | 25600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.96\n",
      "| epoch   5 | 25800/36490 batches | lr 5.00000 | loss  0.72 | ppl     2.06\n",
      "| epoch   5 | 26000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.73\n",
      "| epoch   5 | 26200/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   5 | 26400/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   5 | 26600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 | 26800/36490 batches | lr 5.00000 | loss  0.69 | ppl     1.99\n",
      "| epoch   5 | 27000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.71\n",
      "| epoch   5 | 27200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 | 27400/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   5 | 27600/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 | 27800/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   5 | 28000/36490 batches | lr 5.00000 | loss  0.62 | ppl     1.86\n",
      "| epoch   5 | 28200/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.75\n",
      "| epoch   5 | 28400/36490 batches | lr 5.00000 | loss  0.50 | ppl     1.65\n",
      "| epoch   5 | 28600/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   5 | 28800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.81\n",
      "| epoch   5 | 29000/36490 batches | lr 5.00000 | loss  0.54 | ppl     1.72\n",
      "| epoch   5 | 29200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.87\n",
      "| epoch   5 | 29400/36490 batches | lr 5.00000 | loss  0.51 | ppl     1.66\n",
      "| epoch   5 | 29600/36490 batches | lr 5.00000 | loss  0.48 | ppl     1.62\n",
      "| epoch   5 | 29800/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   5 | 30000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 | 30200/36490 batches | lr 5.00000 | loss  0.56 | ppl     1.76\n",
      "| epoch   5 | 30400/36490 batches | lr 5.00000 | loss  0.45 | ppl     1.56\n",
      "| epoch   5 | 30600/36490 batches | lr 5.00000 | loss  0.49 | ppl     1.64\n",
      "| epoch   5 | 30800/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 | 31000/36490 batches | lr 5.00000 | loss  0.55 | ppl     1.74\n",
      "| epoch   5 | 31200/36490 batches | lr 5.00000 | loss  0.65 | ppl     1.91\n",
      "| epoch   5 | 31400/36490 batches | lr 5.00000 | loss  0.70 | ppl     2.02\n",
      "| epoch   5 | 31600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 31800/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 | 32000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "| epoch   5 | 32200/36490 batches | lr 5.00000 | loss  0.57 | ppl     1.77\n",
      "| epoch   5 | 32400/36490 batches | lr 5.00000 | loss  0.66 | ppl     1.93\n",
      "| epoch   5 | 32600/36490 batches | lr 5.00000 | loss  0.58 | ppl     1.78\n",
      "| epoch   5 | 32800/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.68\n",
      "| epoch   5 | 33000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.83\n",
      "| epoch   5 | 33200/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   5 | 33400/36490 batches | lr 5.00000 | loss  0.71 | ppl     2.04\n",
      "| epoch   5 | 33600/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   5 | 33800/36490 batches | lr 5.00000 | loss  0.59 | ppl     1.80\n",
      "| epoch   5 | 34000/36490 batches | lr 5.00000 | loss  0.60 | ppl     1.82\n",
      "| epoch   5 | 34200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 34400/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.83\n",
      "| epoch   5 | 34600/36490 batches | lr 5.00000 | loss  0.68 | ppl     1.97\n",
      "| epoch   5 | 34800/36490 batches | lr 5.00000 | loss  0.63 | ppl     1.88\n",
      "| epoch   5 | 35000/36490 batches | lr 5.00000 | loss  0.75 | ppl     2.11\n",
      "| epoch   5 | 35200/36490 batches | lr 5.00000 | loss  0.61 | ppl     1.84\n",
      "| epoch   5 | 35400/36490 batches | lr 5.00000 | loss  0.64 | ppl     1.89\n",
      "| epoch   5 | 35600/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   5 | 35800/36490 batches | lr 5.00000 | loss  0.52 | ppl     1.69\n",
      "| epoch   5 | 36000/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.95\n",
      "| epoch   5 | 36200/36490 batches | lr 5.00000 | loss  0.74 | ppl     2.10\n",
      "| epoch   5 | 36400/36490 batches | lr 5.00000 | loss  0.67 | ppl     1.96\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   5 | time: 2035.39s | valid loss  7.62 | valid ppl  2036.16\n",
      "-----------------------------------------------------------------------------------------\n",
      "CPU times: user 33min 27s, sys: 12.2 s, total: 33min 39s\n",
      "Wall time: 33min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for epoch in range(1, n_epochs + 1):\n",
    "    train()\n",
    "    val_loss = evaluate(model, val_data)\n",
    "    print('-' * 89)\n",
    "    print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | valid ppl {:8.2f}'.format(\n",
    "        epoch, time.time() - start_time, val_loss, math.exp(val_loss)))\n",
    "    print('-' * 89)\n",
    "    scheduler.step()  # Atualizar a taxa de aprendizado"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "11ee8948",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-03T16:25:05.913926Z",
     "iopub.status.busy": "2024-10-03T16:25:05.913432Z",
     "iopub.status.idle": "2024-10-03T16:27:49.243051Z",
     "shell.execute_reply": "2024-10-03T16:27:49.240850Z"
    },
    "papermill": {
     "duration": 163.699265,
     "end_time": "2024-10-03T16:27:49.418658",
     "exception": false,
     "start_time": "2024-10-03T16:25:05.719393",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Primeiras previsões do modelo: [array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3])]\n"
     ]
    }
   ],
   "source": [
    "# Função para fazer previsões com o modelo treinado\n",
    "def predict(model, test_data, device):\n",
    "    model.eval()  # Colocar o modelo em modo de avaliação (desliga dropout, batchnorm, etc.)\n",
    "    predictions = []\n",
    "\n",
    "    with torch.no_grad():  # Desligar o cálculo de gradiente, pois não precisamos dele para previsão\n",
    "        for data in test_data:\n",
    "            data = data.to(device)  # Mover os dados para o dispositivo correto (CPU ou GPU)\n",
    "            output = model(data)  # Passar os dados pelo modelo\n",
    "            pred = torch.argmax(output, dim=1)  # Achar a classe com maior probabilidade (para problemas de classificação)\n",
    "            predictions.append(pred.cpu().numpy())  # Mover para CPU e converter para numpy\n",
    "\n",
    "    return predictions\n",
    "\n",
    "# Suponha que 'test_data' seja um DataLoader ou iterador que fornece os dados de teste\n",
    "# Aqui está o uso do modelo para prever as saídas do conjunto de dados de teste\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "test_predictions = predict(model, test_data, device)\n",
    "\n",
    "# Exemplo: Mostrar as primeiras previsões\n",
    "print(\"Primeiras previsões do modelo:\", test_predictions[:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f625345",
   "metadata": {
    "papermill": {
     "duration": 0.172234,
     "end_time": "2024-10-03T16:27:49.763762",
     "exception": false,
     "start_time": "2024-10-03T16:27:49.591528",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59bedd84",
   "metadata": {
    "papermill": {
     "duration": 0.186243,
     "end_time": "2024-10-03T16:27:50.188247",
     "exception": false,
     "start_time": "2024-10-03T16:27:50.002004",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a61d7b4d",
   "metadata": {
    "papermill": {
     "duration": 0.179825,
     "end_time": "2024-10-03T16:27:50.538807",
     "exception": false,
     "start_time": "2024-10-03T16:27:50.358982",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [
    {
     "file_id": "https://github.com/pytorch/tutorials/blob/gh-pages/_downloads/transformer_tutorial.ipynb",
     "timestamp": 1726541081980
    }
   ]
  },
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 5807407,
     "sourceId": 9535015,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30776,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 2363.929568,
   "end_time": "2024-10-03T16:27:52.557332",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-10-03T15:48:28.627764",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}