TriviaQA_to_SQuAD.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMfuNb+FUAqPv5Y0wvknCLL",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Kkordik/NovelQSI/blob/main/TriviaQA_to_SQuAD.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# This is how to convert TriviaQA dataset to the squad format\n",
        "\n",
        "I will be using:\n",
        "https://github.com/mandarjoshi90/triviaqa\n",
        "\n",
        "\n",
        "NOTE, that just running all the cells is not enough, you will need to edit the /content/triviaqa/utils/convert_to_squad_format.py and /content/triviaqa/utils/dataset_utils.py files to let it work.\n",
        "\n",
        "Also before editing check the issue I've opened at github, maybe the problems have been already solved, at the same page I've described how to solve the problems if no changes in github's code were done:\n",
        "https://github.com/mandarjoshi90/triviaqa/issues/9"
      ],
      "metadata": {
        "id": "KwjfNlvd20il"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "Bh3--Dhrdh9d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d2dd4fc-ba38-4c62-8000-5c2ba8221988"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-12-02 10:50:53--  https://nlp.cs.washington.edu/triviaqa/data/triviaqa-rc.tar.gz\n",
            "Resolving nlp.cs.washington.edu (nlp.cs.washington.edu)... 128.208.3.120, 2607:4000:200:12::78\n",
            "Connecting to nlp.cs.washington.edu (nlp.cs.washington.edu)|128.208.3.120|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2665779500 (2.5G) [application/x-gzip]\n",
            "Saving to: ‘triviaqa-rc.tar.gz’\n",
            "\n",
            "triviaqa-rc.tar.gz  100%[===================>]   2.48G  19.7MB/s    in 92s     \n",
            "\n",
            "2023-12-02 10:52:25 (27.7 MB/s) - ‘triviaqa-rc.tar.gz’ saved [2665779500/2665779500]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# @title Install TriviaQA dataset and extract all the dataset files from tar\n",
        "!wget https://nlp.cs.washington.edu/triviaqa/data/triviaqa-rc.tar.gz\n",
        "!tar -xf triviaqa-rc.tar.gz"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Install code from github triviaqa\n",
        "!git clone https://github.com/mandarjoshi90/triviaqa\n",
        "%cd /content/triviaqa"
      ],
      "metadata": {
        "id": "O57wkNkId2Al",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4bb9a145-98e7-4bb9-f2c7-436ed258c0b5"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'triviaqa'...\n",
            "remote: Enumerating objects: 70, done.\u001b[K\n",
            "remote: Counting objects: 100% (3/3), done.\u001b[K\n",
            "remote: Compressing objects: 100% (3/3), done.\u001b[K\n",
            "remote: Total 70 (delta 0), reused 0 (delta 0), pack-reused 67\u001b[K\n",
            "Receiving objects: 100% (70/70), 21.55 KiB | 3.08 MiB/s, done.\n",
            "Resolving deltas: 100% (26/26), done.\n",
            "/content/triviaqa\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Run the converting\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "triviaqa_file = \"/content/triviaqa/qa/wikipedia-dev.json\" # @param {type:\"string\"}\n",
        "squad_file = \"/content/triviaqa/qa/wikipedia-dev-squad1.json\" # @param {type:\"string\"}\n",
        "result_sqad_file = \"/content/dataset_trivia_squad/trivia_qa_squad1.jsonl\"\n",
        "max_num_tokens = 5000 # @param\n",
        "\n",
        "wikipedia_dir = \"/content/triviaqa/evidence/wikipedia\"\n",
        "web_dir = \"/content/triviaqa/evidence/web\"\n",
        "\n",
        "!python /content/triviaqa/convert_to_squad_format.py --triviaqa_file $triviaqa_file --squad_file $squad_file --wikipedia_dir $wikipedia_dir --web_dir $web_dir --max_num_tokens $max_num_tokens"
      ],
      "metadata": {
        "id": "tJlY7u2lgX2Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aee86a87-4ce7-4e1e-d0a5-8888805e799b"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "usage: convert_to_squad_format.py [-h] [--triviaqa_file TRIVIAQA_FILE] [--squad_file SQUAD_FILE]\n",
            "                                  [--wikipedia_dir WIKIPEDIA_DIR] [--web_dir WEB_DIR]\n",
            "                                  [--seed SEED] [--max_num_tokens MAX_NUM_TOKENS]\n",
            "                                  [--sample_size SAMPLE_SIZE] [--tokenizer TOKENIZER]\n",
            "convert_to_squad_format.py: error: argument --triviaqa_file: expected one argument\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Final reformating to real squad format\n",
        "import json\n",
        "\n",
        "# Read the JSON file\n",
        "with open(squad_file, 'r', encoding='utf-8') as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "with open(result_sqad_file, 'a', encoding='utf-8') as fa:\n",
        "  for line in data['data']:\n",
        "    new_line = {\n",
        "        \"context\": line[\"paragraphs\"][0][\"context\"],\n",
        "        \"id\": line[\"paragraphs\"][0][\"qas\"][0][\"id\"],\n",
        "        \"answers\": {\n",
        "            \"answer_start\": [answer[\"answer_start\"] for answer in line[\"paragraphs\"][0][\"qas\"][0][\"answers\"]],\n",
        "            \"text\": [answer[\"text\"] for answer in line[\"paragraphs\"][0][\"qas\"][0][\"answers\"]]\n",
        "            },\n",
        "        \"question\": line[\"paragraphs\"][0][\"qas\"][0][\"question\"]\n",
        "    }\n",
        "    fa.write(json.dumps(new_line) + '\\n')"
      ],
      "metadata": {
        "id": "B5NGEbZpJ7K7"
      },
      "execution_count": 46,
      "outputs": []
    }
  ]
}