Format_GWASdata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format GWAS summary-level data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "library(MRAPSS)\n",
    "library(readr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7055881\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m8\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): MarkerName, Effect_allele, Non_Effect_allele\n",
      "\u001b[32mdbl\u001b[39m (5): Chromosome, Position, Beta, SE, Pvalue\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 7055881 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 6004428 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 5993962 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 5993414 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1148529 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1148507 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1148507 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1148507 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1148507 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1148476 SNPs.\n",
      "\n",
      "The formatted data has 1148476 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 1. AD\n",
    "AD_raw <- readr::read_delim(\"./GWAS_26and5_raw/AD\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "AD = format_data(AD_raw,\n",
    "                 snp_col = \"MarkerName\",\n",
    "                 b_col = \"Beta\",\n",
    "                 se_col = \"SE\",\n",
    "                 A1_col = \"Effect_allele\",\n",
    "                 A2_col = \"Non_Effect_allele\",\n",
    "                 p_col = \"Pvalue\",\n",
    "                 n = 54162)\n",
    "\n",
    "write.table(AD, \"./GWAS_26and5_formatted/AD\", sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"AD\",\"AD_raw\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m11514936\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m9\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): MarkerName, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (6): CHR, POS, EAF_A1, Beta, SE, Pval\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 11514936 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 9791370 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 9742990 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 9661810 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1198814 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1198811 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1084534 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1084534 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1084534 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1084534 SNPs.\n",
      "\n",
      "The formatted data has 1084534 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 2. Alcohol\n",
    "Alcohol_raw <- readr::read_delim(\"./GWAS_26and5_raw/Alcohol\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Alcohol = format_data(Alcohol_raw,\n",
    "                      snp_col = \"MarkerName\",\n",
    "                      b_col = \"Beta\",\n",
    "                      se_col = \"SE\",\n",
    "                      freq_col = \"EAF_A1\",\n",
    "                      A1_col = \"A1\",\n",
    "                      A2_col = \"A2\",\n",
    "                      p_col = \"Pval\",\n",
    "                      n = 414343)\n",
    "\n",
    "write.table(Alcohol, \"./GWAS_26and5_formatted/Alcohol\", sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Alcohol_raw\",\"Alcohol\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10321705\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10321705 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10321675 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8761299 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8711402 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8711402 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161107 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1160815 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037824 SNPs.\n",
      "\n",
      "The formatted data has 1037824 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 3. Angina\n",
    "Angina_raw <- readr::read_delim(\"./GWAS_26and5_raw/Angina\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Angina = format_data(Angina_raw,\n",
    "                     snp_col = \"SNPID_UKB\",\n",
    "                     or_col = \"OR\",\n",
    "                     se_col = \"SE\",\n",
    "                     freq_col = \"MAF_UKB\",\n",
    "                     A1_col = \"A1\",\n",
    "                     A2_col = \"A2\",\n",
    "                     p_col = \"P\",\n",
    "                     n_col = \"NMISS\",\n",
    "                     info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(Angina, \"./GWAS_26and5_formatted/Angina\", sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Angina_raw\",\"Angina\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m8219102\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m14\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (4): ID, REF, ALT, DIRE\n",
      "\u001b[32mdbl\u001b[39m (10): CHROM, POS, BETA, SE, PVAL, NGT, IMPINFO, NEFFDIV2, NCAS, NCON\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 8219102 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 5880021 SNPs.\n",
      "\n",
      "effect allele column has some values that are not A/C/T/G. Remove these SNPs..., remaining 5880020 SNPs.\n",
      "\n",
      "the other allele column has some values that are not A/C/T/G. Remove these SNPs..., remaining 5880019 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 5064495 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 5033100 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 4986546 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 922421 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 922420 SNPs.\n",
      "\n",
      "Generating sample size from ncase and ncontrol\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 922420 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 922420 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 910374 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 910374 SNPs.\n",
      "\n",
      "The formatted data has 910374 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 4. Anorexia\n",
    "Anorexia_raw <- readr::read_delim(\"./GWAS_26and5_raw/Anorexia\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Anorexia = format_data(Anorexia_raw,\n",
    "                       snp_col = \"ID\",\n",
    "                       b_col = \"BETA\",\n",
    "                       se_col = \"SE\",\n",
    "                       A1_col = \"ALT\",\n",
    "                       A2_col = \"REF\",\n",
    "                       p_col = \"PVAL\",\n",
    "                       ncase_col  = \"NCAS\",\n",
    "                       ncontrol_col = \"NCON\",\n",
    "                       info_col = \"IMPINFO\")\n",
    "\n",
    "write.table(Anorexia, \"./GWAS_26and5_formatted/Anorexia\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Anorexia_raw\",\"Anorexia\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m6517324\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m15\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (4): snp, A1, A2, direction\n",
      "\u001b[32mdbl\u001b[39m (11): chr, bp_hg19, or, tor_l95, or_u95, b, se, p, freq_A1, info, N\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 6517324 dat lines\n",
      "\n",
      "effect allele column has some values that are not A/C/T/G. Remove these SNPs..., remaining 6517323 SNPs.\n",
      "\n",
      "the other allele column has some values that are not A/C/T/G. Remove these SNPs..., remaining 6517322 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 5593524 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 5585718 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 5585718 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1093312 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1079411 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1079411 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1076873 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1076873 SNPs.\n",
      "\n",
      "The formatted data has 1076873 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 5. ASD\n",
    "ASD_raw <- readr::read_delim(\"./GWAS_26and5_raw/ASD\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "ASD = format_data(ASD_raw,\n",
    "                  snp_col = \"snp\",\n",
    "                  b_col = \"b\",\n",
    "                  se_col = \"StdErr\",\n",
    "                  freq_col = \"freq_A1\",\n",
    "                  A1_col = \"A1\",\n",
    "                  A2_col = \"A2\",\n",
    "                  p_col = \"p\",\n",
    "                  n_col = \"N\")\n",
    "\n",
    "write.table(ASD, \"./GWAS_26and5_formatted/ASD\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"ASD_raw\",\"ASD\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, BETA, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UK...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037799 SNPs.\n",
      "\n",
      "The formatted data has 1037799 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 6. BMI\n",
    "BMI_raw <- readr::read_delim(\"./GWAS_26and5_raw/BMI\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "BMI = format_data(BMI_raw,\n",
    "                  snp_col = \"SNPID_UKB\",\n",
    "                  b_col = \"BETA\",\n",
    "                  se_col = \"SE\",\n",
    "                  freq_col = \"MAF_UKB\",\n",
    "                  A1_col = \"A1\",\n",
    "                  A2_col = \"A2\",\n",
    "                  p_col = \"P\",\n",
    "                  n_col = \"NMISS\",\n",
    "                  info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(BMI, \"./GWAS_26and5_formatted/BMI\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"BMI_raw\",\"BMI\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m9455778\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m13\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (4): markername, effect_allele, noneffect_allele, model\n",
      "\u001b[32mdbl\u001b[39m (9): chr, bp_hg19, effect_allele_freq, median_info, beta, se_dgc, p_dgc,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 9455778 dat lines\n",
      "\n",
      "effect allele column has some values that are not A/C/T/G. Remove these SNPs..., remaining 9455777 SNPs.\n",
      "\n",
      "the other allele column has some values that are not A/C/T/G. Remove these SNPs..., remaining 9455776 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8123051 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8109390 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8109390 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1192337 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1112984 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1112984 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1112984 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1112977 SNPs.\n",
      "\n",
      "The formatted data has 1112977 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 7. CAD\n",
    "CAD_raw <- readr::read_delim(\"./GWAS_26and5_raw/CAD\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "CAD = format_data(CAD_raw,\n",
    "                  snp_col = \"markername\",\n",
    "                  b_col = \"beta\",\n",
    "                  se_col = \"se\",\n",
    "                  freq_col = \"effect_allele_freq\",\n",
    "                  A1_col = \"effect_allele\",\n",
    "                  A2_col = \"noneffect_allele\",\n",
    "                  p_col = \"p_dgc\",\n",
    "                  n = 184305)\n",
    "\n",
    "write.table(CAD, \"./GWAS_26and5_formatted/CAD\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"CAD_raw\",\"CAD\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7055380\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): rsid, Allele1, Allele2\n",
      "\u001b[32mdbl\u001b[39m (4): freq, Effect, StdErr, P.value\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 7055380 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 6012676 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 5983174 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 5979640 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1108128 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1108106 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1004931 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1004931 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1004931 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1004565 SNPs.\n",
      "\n",
      "The formatted data has 1004565 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 8. CD (Crohn Disease)\n",
    "# CD_rsid: CD annotated with rs-number and freq\n",
    "\n",
    "CD_raw <- readr::read_delim(\"./GWAS_26and5_raw/CD_rsid\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "CD = format_data(CD_raw,\n",
    "                 snp_col = \"rsid\",\n",
    "                 b_col = \"Effect\",\n",
    "                 se_col = \"StdErr\",\n",
    "                 freq_col = \"freq\",\n",
    "                 A1_col = \"Allele1\",\n",
    "                 A2_col = \"Allele2\",\n",
    "                 p_col = \"P.value\",\n",
    "                 n = 40266)\n",
    "\n",
    "write.table(CD, \"./GWAS_26and5_formatted/CD\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"CD_raw\",\"CD\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m14661601\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): SNP, ALLELE1, ALLELE0\n",
      "\u001b[32mdbl\u001b[39m (7): CHR, BP, A1FREQ, INFO, BETA, SE, P\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 14661601 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10903657 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 9250294 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 9202639 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 9196675 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1198480 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1198465 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1065141 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1065141 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1065141 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1065141 SNPs.\n",
      "\n",
      "The formatted data has 1065141 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 9. DaytimeSleepiness\n",
    "dat_raw <- readr::read_delim(\"./GWAS_26and5_raw/DaytimeSleepiness\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "dat = format_data(dat_raw,\n",
    "                  snp_col = \"SNP\",\n",
    "                  b_col = \"BETA\",\n",
    "                  se_col = \"SE\",\n",
    "                  freq_col = \"A1FREQ\",\n",
    "                  A1_col = \"ALLELE1\",\n",
    "                  A2_col = \"ALLELE0\",\n",
    "                  info_col = \"INFO\",\n",
    "                  p_col = \"P\",\n",
    "                  n = 452071)\n",
    "\n",
    "write.table(dat, \"./GWAS_26and5_formatted/Daytime_Sleepiness\",  sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"dat_raw\",\"dat\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10886529\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m12\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (4): RSID, SNP, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (8): CHR, POS, EAF_UKB, MAF_UKB, Z, P, N, INFO_UKB\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10886529 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10847393 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 9204753 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 9152908 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 9152468 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161303 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161185 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1026902 SNPs.\n",
      "\n",
      "Remove missing values, remaining 1026902 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1026744 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1026744 SNPs.\n",
      "\n",
      "The formatted data has 1026744 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 10. Depression\n",
    "dat_raw <- readr::read_delim(\"./GWAS_26and5_raw/Depression\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "dat = MRAPSS::format_data(dat_raw,\n",
    "                          snp_col = \"RSID\",\n",
    "                          z_col = \"Z\",\n",
    "                          freq_col = \"MAF_UKB\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          p_col = \"P\",\n",
    "                          n_col = \"N\",\n",
    "                          info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(dat, \"./GWAS_26and5_formatted/Depression\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"dat_raw\",\"dat\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037702 SNPs.\n",
      "\n",
      "The formatted data has 1037702 dat lines. \n",
      "\n",
      "\n",
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037533 SNPs.\n",
      "\n",
      "The formatted data has 1037533 dat lines. \n",
      "\n",
      "\n",
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037381 SNPs.\n",
      "\n",
      "The formatted data has 1037381 dat lines. \n",
      "\n",
      "\n",
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037742 SNPs.\n",
      "\n",
      "The formatted data has 1037742 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 11-14. Hair_Black, Hair_Blonde, Hair_Dark_Brown, Hair_Light_Brown\n",
    "for (trait in c(\"Hair_Black\", \"Hair_Blonde\", \"Hair_Dark_Brown\", \"Hair_Light_Brown\")){\n",
    "  \n",
    "  dat_raw <- readr::read_delim(paste0(\"./GWAS_26and5_raw/\", trait), delim = \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "  \n",
    "  dat = format_data(dat_raw,\n",
    "                    snp_col = \"SNPID_UKB\",\n",
    "                    or_col = \"OR\",\n",
    "                    se_col = \"SE\",\n",
    "                    freq_col = \"MAF_UKB\",\n",
    "                    A1_col = \"A1\",\n",
    "                    A2_col = \"A2\",\n",
    "                    p_col = \"P\",\n",
    "                    n_col = \"NMISS\",\n",
    "                    info_col = \"INFO_UKB\")\n",
    "  \n",
    "  write.table(dat, paste0(\"./GWAS_26and5_formatted/\", trait), sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "  \n",
    "  rm(list=c(\"dat_raw\", \"dat\"))\n",
    "  \n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037824 SNPs.\n",
      "\n",
      "The formatted data has 1037824 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 15. HBP (High Blood Pressure)\n",
    "\n",
    "HBP_raw <- readr::read_delim(\"./GWAS_26and5_raw/HBP\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "HBP = MRAPSS::format_data(HBP_raw,\n",
    "                          snp_col = \"SNPID_UKB\",\n",
    "                          or_col = \"OR\",\n",
    "                          se_col = \"SE\",\n",
    "                          freq_col = \"MAF_UKB\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          p_col = \"P\",\n",
    "                          n_col = \"NMISS\",\n",
    "                          info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(HBP, \"./GWAS_26and5_formatted/HBP\", sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"HBP_raw\", \"HBP\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m2550858\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m8\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): MarkerName, Allele1, Allele2\n",
      "\u001b[32mdbl\u001b[39m (5): Freq.Allele1.HapMapCEU, b, SE, p, N\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 2547281 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 2154206 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 2143731 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 2143731 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1091189 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1091150 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 975042 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 975042 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 961175 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 961042 SNPs.\n",
      "\n",
      "The formatted data has 961042 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 16. Height (GIANT)\n",
    "\n",
    "Height_raw <- readr::read_delim(\"./GWAS_26and5_raw/Height_GIANT\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "Height_raw = Height_raw[!is.na(Height_raw$Freq.Allele1.HapMapCEU), ]\n",
    "Height = format_data(Height_raw,\n",
    "                     snp_col = \"MarkerName\",\n",
    "                     b_col = \"b\",\n",
    "                     se_col = \"SE\",\n",
    "                     freq_col = \"Freq.Allele1.HapMapCEU\",\n",
    "                     A1_col = \"Allele1\",\n",
    "                     A2_col = \"Allele2\",\n",
    "                     p_col = \"p\",\n",
    "                     n_col = \"N\")\n",
    "\n",
    "write.table(Height, \"./GWAS_26and5_formatted/Height_GIANT\", sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Height_raw\",\"Height\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10599054\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, BETA, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UK...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10599054 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10599018 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8995068 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8944044 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8944044 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161501 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161195 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037320 SNPs.\n",
      "\n",
      "The formatted data has 1037320 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 17. Height (UKBB)\n",
    "\n",
    "Height_UKB_raw <- readr::read_delim(\"./GWAS_26and5_raw/Height_UKB\", \"\\t\", escape_double = FALSE,\n",
    "                                    trim_ws = TRUE, progress = F)\n",
    "\n",
    "Height_UKB = format_data(Height_UKB_raw,\n",
    "                         snp_col = \"SNPID_UKB\",\n",
    "                         b_col = \"BETA\",\n",
    "                         se_col = \"SE\",\n",
    "                         freq_col = \"MAF_UKB\",\n",
    "                         A1_col = \"A1\",\n",
    "                         A2_col = \"A2\",\n",
    "                         p_col = \"P\",\n",
    "                         n_col = \"NMISS\",\n",
    "                         info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(Height_UKB, \"./GWAS_26and5_formatted/Height_UKB\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Height_UKB_raw\",\"Height_UKB\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7059162\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): rsid, Allele1, Allele2\n",
      "\u001b[32mdbl\u001b[39m (4): freq, Effect, StdErr, P.value\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 7059162 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 6015921 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 5986496 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 5982838 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1108191 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1108169 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1004923 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1004923 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1004923 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1004597 SNPs.\n",
      "\n",
      "The formatted data has 1004597 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 18. IBD (Inflammatory Bowel Disease)\n",
    "\n",
    "IBD_raw <- readr::read_delim(\"./GWAS_26and5_raw/IBD_rsid\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "IBD = format_data(IBD_raw,\n",
    "                  snp_col = \"rsid\",\n",
    "                  b_col = \"Effect\",\n",
    "                  se_col = \"StdErr\",\n",
    "                  freq_col = \"freq\",\n",
    "                  A1_col = \"Allele1\",\n",
    "                  A2_col = \"Allele2\",\n",
    "                  p_col = \"P.value\",\n",
    "                  n = 59957)\n",
    "\n",
    "write.table(IBD, \"./GWAS_26and5_formatted/IBD\", sep=\"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"IBD_raw\",\"IBD\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m18485882\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m8\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \" \"\n",
      "\u001b[31mchr\u001b[39m (3): SNP, Non_effect_Allele, Effect_Allele\n",
      "\u001b[32mdbl\u001b[39m (5): Chr, BPos, Beta, Standard_Error_of_Beta, P\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 18485882 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 15709024 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 15648261 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 15648261 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1208817 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1208814 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1208814 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1208814 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1208814 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1208814 SNPs.\n",
      "\n",
      "The formatted data has 1208814 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 19. Income\n",
    "\n",
    "Income_raw <- readr::read_delim(\"./GWAS_26and5_raw/Income\", \" \", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Income = format_data(Income_raw,\n",
    "                     snp_col = \"SNP\",\n",
    "                     b_col = \"Beta\",\n",
    "                     se_col = \"Standard_Error_of_Beta\",\n",
    "                     A1_col = \"Effect_Allele\",\n",
    "                     A2_col = \"Non_effect_Allele\",\n",
    "                     p_col = \"P\",\n",
    "                     n=286301)\n",
    "\n",
    "write.table(Income, \"./GWAS_26and5_formatted/Income\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Income_raw\",\"Income\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m14661601\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \" \"\n",
      "\u001b[31mchr\u001b[39m (3): SNP, ALLELE1, ALLELE0\n",
      "\u001b[32mdbl\u001b[39m (7): CHR, BP, A1FREQ, INFO, BETA_INSOMNIA, SE_INSOMNIA, P_INSOMNIA\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 14661601 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10903657 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 9250294 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 9202639 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 9196675 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1198480 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1198465 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1065149 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1065149 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1065149 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1065149 SNPs.\n",
      "\n",
      "The formatted data has 1065149 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 20. Insomnia\n",
    "\n",
    "Insomnia_raw <- readr::read_delim(\"./GWAS_26and5_raw/Insomnia\", \" \", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Insomnia = format_data(Insomnia_raw,\n",
    "                       snp_col = \"SNP\",\n",
    "                       b_col = \"BETA_INSOMNIA\",\n",
    "                       se_col = \"SE_INSOMNIA\",\n",
    "                       freq_col = \"A1FREQ\",\n",
    "                       A1_col = \"ALLELE1\",\n",
    "                       A2_col = \"ALLELE0\",\n",
    "                       info_col = \"INFO\",\n",
    "                       p_col = \"P_INSOMNIA\",\n",
    "                       n = 453379)\n",
    "\n",
    "write.table(Insomnia, \"./GWAS_26and5_formatted/Insomnia\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Insomnia_raw\",\"Insomnia\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m9561902\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, BETA, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UK...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 9561902 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 9561873 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8119324 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8072530 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8072530 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1159117 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1158862 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037824 SNPs.\n",
      "\n",
      "The formatted data has 1037824 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 21. Intelligence\n",
    "\n",
    "Intelligence_raw <- readr::read_delim(\"./GWAS_26and5_raw/Intelligence\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Intelligence = format_data(Intelligence_raw,\n",
    "                           snp_col = \"SNPID_UKB\",\n",
    "                           b_col = \"BETA\",\n",
    "                           se_col = \"SE\",\n",
    "                           freq_col = \"MAF_UKB\",\n",
    "                           A1_col = \"A1\",\n",
    "                           A2_col = \"A2\",\n",
    "                           p_col = \"P\",\n",
    "                           n_col = \"NMISS\",\n",
    "                           info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(Intelligence, \"./GWAS_26and5_formatted/Intelligence\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Intelligence_raw\",\"Intelligence\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10154467\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, OR, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UKB,...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10154467 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10154437 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8620374 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8571258 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8571258 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1160811 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1160529 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037824 SNPs.\n",
      "\n",
      "The formatted data has 1037824 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 22. MDD\n",
    "MDD_raw <- readr::read_delim(\"./GWAS_26and5_raw/MDD\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "MDD = MRAPSS::format_data(MDD_raw,\n",
    "                          snp_col = \"SNPID_UKB\",\n",
    "                          or_col = \"OR\",\n",
    "                          se_col = \"SE\",\n",
    "                          freq_col = \"MAF_UKB\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          p_col = \"P\",\n",
    "                          n_col = \"NMISS\",\n",
    "                          info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(MDD, \"./GWAS_26and5_formatted/MDD\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"MDD_raw\",\"MDD\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m2474037\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m8\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): SNPID, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (5): CHR, POS, Freq_HapMap, Zscore, Pvalue\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 2474037 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 2092463 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 2089374 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 2089374 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1078165 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 984184 SNPs.\n",
      "\n",
      "Remove missing values, remaining 984184 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 984184 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 984184 SNPs.\n",
      "\n",
      "The formatted data has 984184 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 23. NEB\n",
    "NEB_raw <- readr::read_delim(\"./GWAS_26and5_raw/NEB\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "NEB = MRAPSS::format_data(NEB_raw,\n",
    "                          snp_col = \"SNPID\",\n",
    "                          freq_col = \"Freq_HapMap\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          z_col = \"Zscore\",\n",
    "                          p_col = \"Pvalue\",\n",
    "                          n = 343072)\n",
    "\n",
    "write.table(NEB, file = \"./GWAS_26and5_formatted/NEB\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"NEB_raw\",\"NEB\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10399545\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, BETA, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UK...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10399545 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10399514 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8826898 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8776612 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8776612 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161220 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1160924 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037824 SNPs.\n",
      "\n",
      "The formatted data has 1037824 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 24. Neuroticism\n",
    "dat_raw <- readr::read_delim(\"./GWAS_26and5_raw/Neuroticism\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "dat = MRAPSS::format_data(dat_raw,\n",
    "                          snp_col = \"SNPID_UKB\",\n",
    "                          b_col = \"BETA\",\n",
    "                          se_col = \"SE\",\n",
    "                          freq_col = \"MAF_UKB\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          p_col = \"P\",\n",
    "                          n_col = \"NMISS\",\n",
    "                          info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(dat, \"./GWAS_26and5_formatted/Neuroticism\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"dat_raw\",\"dat\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7022577\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): SNPID, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (7): freq, Chr, Position(hg19), OR(A1), OR_95%CIlow, OR_95%CIup, P-val\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 7022577 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 5963060 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 5935712 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 5935712 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1131678 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1030014 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1030014 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1030014 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1029912 SNPs.\n",
      "\n",
      "The formatted data has 1029912 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 25. RA\n",
    "# RA_freq: RA annotated with a column of freq\n",
    "# Note that the freq_col is not a required column. If freq_col is not available, it will skip the step of QC with freq.\n",
    "\n",
    "RA_raw <- readr::read_delim(\"./GWAS_26and5_raw/RA_freq\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "RA = format_data(RA_raw,\n",
    "                  snp_col = \"SNPID\",\n",
    "                  or_col = \"OR(A1)\",\n",
    "                  A1_col = \"A1\",\n",
    "                  A2_col = \"A2\",\n",
    "                  freq_col = \"freq\",\n",
    "                  p_col = \"P-val\",\n",
    "                  n = 58284)\n",
    "\n",
    "write.table(RA, \"./GWAS_26and5_formatted/RA\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"RA_raw\",\"RA\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7480977\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (4): rsid, SNP, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (6): Freq.A1, CHR, BP, OR, SE, P\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 7480977 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 6341312 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 6315985 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 6315867 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1145148 SNPs.\n",
      "\n",
      "infer b column from log(or)...\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1047566 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1047566 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1047566 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1047564 SNPs.\n",
      "\n",
      "The formatted data has 1047564 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 26. SCZ\n",
    "\n",
    "SCZ_raw <- readr::read_delim(\"./GWAS_26and5_raw/SCZ_rsid\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "SCZ = format_data(SCZ_raw,\n",
    "                  snp_col = \"rsid\",\n",
    "                  or_col = \"OR\",\n",
    "                  se_col = \"SE\",\n",
    "                  freq_col = \"Freq.A1\",\n",
    "                  A1_col = \"A1\",\n",
    "                  A2_col = \"A2\",\n",
    "                  p_col = \"P\",\n",
    "                  n=105318)\n",
    "\n",
    "write.table(SCZ, \"./GWAS_26and5_formatted/SCZ\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"SCZ_raw\",\"SCZ\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m13933175\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m13\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (3): RSID, REF, ALT\n",
      "\u001b[32mdbl\u001b[39m (10): CHROM, POS, AF, STAT, PVALUE, BETA, SE, N, EFFECTIVE_N, Number_of_...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 12168248 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 10357018 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 10306304 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 10298519 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1189942 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1073202 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1073202 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1043420 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1043420 SNPs.\n",
      "\n",
      "The formatted data has 1043420 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 27. Smoking\n",
    "Smoking_raw <- readr::read_delim(\"./GWAS_26and5_raw/Smoking\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "Smoking_raw = Smoking_raw[!is.na(Smoking_raw$AF), ]\n",
    "Smoking = format_data(Smoking_raw,\n",
    "                      snp_col = \"RSID\",\n",
    "                      b_col = \"BETA\",\n",
    "                      se_col = \"SE\",\n",
    "                      freq_col = \"AF\",\n",
    "                      A1_col = \"ALT\",\n",
    "                      A2_col = \"REF\",\n",
    "                      p_col = \"PVALUE\",\n",
    "                      n_col = \"N\")\n",
    "\n",
    "write.table(Smoking , \"./GWAS_26and5_formatted/Smoking\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Smoking_raw\",\"Smoking\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m2268674\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m9\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m (3): MarkerName, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (6): CHR, POS, EAF, Beta, SE, Pval\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 2268674 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 1918560 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 1915787 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 1915787 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1018811 SNPs.\n",
      "\n",
      "Generating sample size from specified sample size\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 975109 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 975109 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 975109 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 975109 SNPs.\n",
      "\n",
      "The formatted data has 975109 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 28. SWB\n",
    "\n",
    "SWB_raw <- readr::read_delim(\"./GWAS_26and5_raw/SWB\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "SWB = MRAPSS::format_data(SWB_raw,\n",
    "                          snp_col = \"MarkerName\",\n",
    "                          freq_col = \"EAF\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          b_col = \"Beta\",\n",
    "                          se_col = \"SE\",\n",
    "                          p_col = \"Pval\",\n",
    "                          n = 298420)\n",
    "\n",
    "write.table(SWB, file = \"./GWAS_26and5_formatted/SWB\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"SWB_raw\",\"SWB\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m9073559\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m11\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (3): rsid, EA, NEA\n",
      "\u001b[32mdbl\u001b[39m  (7): Chr, Pos, EAF, Beta, SE, Pvalue, Neff\n",
      "\u001b[34mtime\u001b[39m (1): SNP\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 9073559 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 7713357 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 7671570 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 7663228 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1186472 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1058930 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1058930 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1048396 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1048310 SNPs.\n",
      "\n",
      "The formatted data has 1048310 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 29. T2D\n",
    "\n",
    "T2D_raw <- readr::read_delim(\"./GWAS_26and5_raw/T2D_rsid\", \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "T2D = MRAPSS::format_data(T2D_raw,\n",
    "                          snp_col = \"rsid\",\n",
    "                          b_col = \"Beta\",\n",
    "                          se_col = \"SE\",\n",
    "                          freq_col = \"EAF\",\n",
    "                          A1_col = \"EA\",\n",
    "                          A2_col = \"NEA\",\n",
    "                          p_col = \"Pvalue\",\n",
    "                          n_col = \"Neff\")\n",
    "\n",
    "write.table(T2D, \"./GWAS_26and5_formatted/T2D\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"T2D_raw\",\"T2D\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m10579925\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m20\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n",
      "\u001b[31mchr\u001b[39m  (7): SNP, A1, TEST, A2, SNPID_UKB, A1_UKB, A2_UKB\n",
      "\u001b[32mdbl\u001b[39m (13): CHR, BP, NMISS, BETA, SE, L95, U95, STAT, P, MAF, NCHROBS, INFO_UK...\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 10579925 dat lines\n",
      "\n",
      "Remove SNPs with imputation info less than 0.9 ..., remaining 10579890 SNPs.\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 8978911 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 8927942 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 8927942 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1161467 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1161162 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1037824 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1037824 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1037026 SNPs.\n",
      "\n",
      "The formatted data has 1037026 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 30. Tanning\n",
    "\n",
    "dat_raw <- readr::read_delim(\"./GWAS_26and5_raw/Tanning\", delim = \"\\t\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "dat = MRAPSS::format_data(dat_raw,\n",
    "                          snp_col = \"SNPID_UKB\",\n",
    "                          b_col = \"BETA\",\n",
    "                          se_col = \"SE\",\n",
    "                          freq_col = \"MAF_UKB\",\n",
    "                          A1_col = \"A1\",\n",
    "                          A2_col = \"A2\",\n",
    "                          p_col = \"P\",\n",
    "                          n_col = \"NMISS\",\n",
    "                          info_col = \"INFO_UKB\")\n",
    "\n",
    "write.table(dat, \"./GWAS_26and5_formatted/Tanning\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"dat_raw\", \"dat\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mRows: \u001b[1m\u001b[22m\u001b[34m\u001b[34m2450547\u001b[34m\u001b[39m \u001b[1m\u001b[1mColumns: \u001b[1m\u001b[22m\u001b[34m\u001b[34m7\u001b[34m\u001b[39m\n",
      "\n",
      "\u001b[36m──\u001b[39m \u001b[1m\u001b[1mColumn specification\u001b[1m\u001b[22m \u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[39m\n",
      "\u001b[1mDelimiter:\u001b[22m \",\"\n",
      "\u001b[31mchr\u001b[39m (3): MarkerName, A1, A2\n",
      "\u001b[32mdbl\u001b[39m (4): n_total, beta, se, p_gc\n",
      "\n",
      "\n",
      "\u001b[36mℹ\u001b[39m Use \u001b[30m\u001b[47m\u001b[30m\u001b[47m`spec()`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to retrieve the full column specification for this data.\n",
      "\u001b[36mℹ\u001b[39m Specify the column types or set \u001b[30m\u001b[47m\u001b[30m\u001b[47m`show_col_types = FALSE`\u001b[47m\u001b[30m\u001b[49m\u001b[39m to quiet this message.\n",
      "\n",
      "Begin formatting .... \n",
      "\n",
      "The raw data set has 2450547 dat lines\n",
      "\n",
      "Remove ambiguous SNPs ..., remaining 2072342 SNPs.\n",
      "\n",
      "Remove SNPs in MHC region ..., remaining 2062309 SNPs.\n",
      "\n",
      "Remove duplicated SNPs ..., remaining 2062309 SNPs.\n",
      "\n",
      "Merge SNPs with the hapmap3 snplist ..., remaining 1064399 SNPs.\n",
      "\n",
      "Remove SNPs with alleles not matched with the hapmap3 snplist, remaining 1064363 SNPs.\n",
      "\n",
      "Remove SNPs with p-value < 0 or p-value > 1, remaining 1064363 SNPs.\n",
      "\n",
      "Inferring z score from p value and b ...\n",
      "\n",
      "Remove missing values, remaining 1064363 SNPs.\n",
      "\n",
      "Remove SNPs with sample size 5 standard deviations away from the mean, remaining 1060694 SNPs.\n",
      "\n",
      "Remove SNPs with chi2 > chi2_max ... , remaining 1060305 SNPs.\n",
      "\n",
      "The formatted data has 1060305 dat lines. \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 31. Urate\n",
    "\n",
    "Urate_raw <- readr::read_delim(\"./GWAS_26and5_raw/Urate\", \",\", escape_double = FALSE, trim_ws = TRUE, progress = F)\n",
    "\n",
    "Urate = format_data(Urate_raw,\n",
    "                    snp_col = \"MarkerName\",\n",
    "                    b_col = \"beta\",\n",
    "                    se_col = \"se\",\n",
    "                    A1_col = \"A1\",\n",
    "                    A2_col = \"A2\",\n",
    "                    p_col = \"p_gc\",\n",
    "                    n_col = \"n_total\")\n",
    "\n",
    "write.table(Urate, \"./GWAS_26and5_formatted/Urate\", sep = \"\\t\", quote = F, row.names = F, col.names = T)\n",
    "\n",
    "rm(list=c(\"Urate_raw\",\"Urate\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}