Final Generalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imported libraries\n",
    "import pydicom\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sys\n",
    "import matplotlib as mpl\n",
    "import imageio\n",
    "import cv2\n",
    "import sys\n",
    "import os\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Place directory paths and declare needed\n",
    "# variables for initiating the general categorization\n",
    "# and storing of patient scans arrays\n",
    "\n",
    "original_directory_path = '/media/omar/New Volume/TCIA Data/GBM/Data/TCGA-GBM/'\n",
    "segmented_directory_path = '/home/omar/Desktop/DropboxData/Dropbox-GBM/TCGA-GBM/'\n",
    "result_directory_path = '/home/omar/Desktop/TrainingResult_dup/'\n",
    "patients = []\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "Patient pickle file structure\n",
    "File Name: Patient ID (TCGA-xx-xxxx)\n",
    "DATA:\n",
    "[\n",
    "    [sorted original 23 arrays of AX T2],\n",
    "    [sorted original 23 arrays of AX T1],\n",
    "    [sorted original 23 arrays of FLAIR],\n",
    "    [sorted original 23 arrays of POST],\n",
    "    [segmentation array]\n",
    "\n",
    "]\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "# Get all the patient IDs that will be used for the training phase and sort them\n",
    "for patient in os.listdir(segmented_directory_path):\n",
    "    patients.append(patient)\n",
    "patients.sort()\n",
    "# We now have a list of all the patients needed for the training phase\n",
    "# print (patients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'----------------------------------------'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problems log, updates and status\n",
    "\"\"\"\n",
    "    \n",
    "    Retreiving files of the original patient scans for designated scan\n",
    "    Problems occured:\n",
    "    1- Not all patient directories have single scan history\n",
    "    Patients with such issue are: 06-0138, 19-5960.\n",
    "    Resolved by validating ID for every scan\n",
    "    Status: Resolved by defining an aiding module 'multipleVisits' that operates in the else statement\n",
    "    \n",
    "    if(one visit)-> undergo the process of retreiving patient scans images\n",
    "    else -> use the 'multipleVisits' module to identify the visit date containing scans\n",
    "            segmented followed by retreiving patient data\n",
    "                \n",
    "    Another problem to be handled\n",
    "    2- Second problem is the sorting of the original scans dicom files.\n",
    "    The files are named 0000-0023. However they aren't aligned in order through\n",
    "    that naming, instead there is a dicom object attribute called\n",
    "    InstanceNumber that identifies the order of the scans.\n",
    "    Status: An aiding function 'dicomSort' was declared to handle this issue.\n",
    "                \n",
    "    \n",
    "    A third problem occured\n",
    "    3- Some of the original scans aren't of the usual dimensions (256,256)\n",
    "    but rather (512,512). Problem discovered at\n",
    "    Patient: TCGA-06-0138\n",
    "    Visit Date: 11-25-2002-91840\n",
    "    Scan: 10-AX T1 POST GD FLAIR-87338\n",
    "    Dicom: 000000.dcm\n",
    "    UNKNOWN IF SAME ISSUE OCCURS AT OTHER PLACES ALTHOUGH ITS LIKELY FOR THE SAME PATIENT\n",
    "    \n",
    "    Problem discovered when storing the original scan arrays in an already\n",
    "    initialized original_scans npArray of zeroes that was initialized with\n",
    "    the dimensions (256,256)\n",
    "    --------------------------------------------------------------------------\n",
    "    Status: function imageArray declared in the testing area was supposed to handle this\n",
    "    but halted for now.\n",
    "                \n",
    "    \n",
    "    \"\"\"\n",
    "\"----------------------------------------\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aiding modules and functions\n",
    "\"\"\"\n",
    "    Module to handle the existence of subfolders for multiple patient visits\n",
    "    Purpose and working principle:\n",
    "    Iterates over the original patient subfolders till it finds scan folders relevant by scan ID\n",
    "    Inputs are, patient ID and an array containing\n",
    "    the segmentation filenames in the form 'seg_corrected_AX-*****.dcm'\n",
    "    Returns the subfolders containing the relevant original scans (usually one folder)\"\"\"\n",
    "\n",
    "def multipleVisits(patient_id, segmented_scans):\n",
    "    subfolders = os.listdir(original_directory_path+patient_id)\n",
    "    segmented_ids = []\n",
    "    for i in segmented_scans:\n",
    "        segmented_ids.append(i[-9:-4])\n",
    "    target_folders = []\n",
    "    for folder in subfolders:\n",
    "        images_folder = os.listdir(original_directory_path+patient_id+\"/\"+folder)\n",
    "        for scans in images_folder:\n",
    "            if scans[-5:] in segmented_ids and folder not in target_folders:\n",
    "                target_folders.append(folder)\n",
    "    return target_folders\n",
    "\n",
    "\"\"\"\n",
    "    Module to handle the issue with sorting of the original dicom files of the\n",
    "    patient because the filenames are in no way sorted either. The working principle\n",
    "    of the function is that it takes in an array of filenames and returns a dictionary\n",
    "    where key is the instance number and the value is the correspondent file name.\n",
    "    The keys are sorted to make the appending the original patient data in the pickle \n",
    "    file be in the correct order\n",
    "\"\"\"\n",
    "\n",
    "def dicomSort(path_to_scans, file_array):\n",
    "    dicom_sorting = {}\n",
    "    for file in range(len(file_array)):\n",
    "        instance = pydicom.dcmread(path_to_scans + \"/\" + file_array[file])\n",
    "        instance_value = instance.InstanceNumber\n",
    "        dicom_sorting[instance_value] = file_array[file]\n",
    "    sorted(dicom_sorting.keys())\n",
    "    return dicom_sorting\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "    Module to traverse the scan segmentation dicom file and export slice arrays\n",
    "    to be added to the respective patient pickle data file. It should be generic\n",
    "    and not limited to only 23 slices after learning that not all patients have 23\n",
    "    slices only.\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def segmentationArray(path_to_scan, file):\n",
    "    ds = pydicom.dcmread(path_to_scan+\"/\"+file).pixel_array\n",
    "    number_of_slices = ds.shape[0]\n",
    "    big_range = (int)(number_of_slices/3)   \n",
    "    tumor_core = ds[0:big_range,:,:]\n",
    "    edema = ds[big_range:(big_range*2),:,:]\n",
    "    non_enhanced_tumor = ds[(big_range*2):,:,:]\n",
    "    unified_label = tumor_core + edema + non_enhanced_tumor\n",
    "    unified_label = np.swapaxes(unified_label,0,2)\n",
    "    \n",
    "    return unified_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "\n",
    "After having the training patients recognized, we now need to start\n",
    "creating pickle files for every patient to store the data at\n",
    "\n",
    "Pickle file Information\n",
    "Name: Patient\n",
    "Data included:\n",
    "a. original scan images\n",
    "b. arrays for each of the segmented scans\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "os.chdir(result_directory_path)\n",
    "\n",
    "for patient_id in patients:\n",
    "#     Getting the corrected segmented scan filenames for every patient (ID at indexing -9:-4)\n",
    "    patient_segmented_files = []\n",
    "    for scan in os.listdir(segmented_directory_path+patient_id+'/DSO/'):\n",
    "        if \"corrected\" in scan:\n",
    "            patient_segmented_files.append(scan)\n",
    "    \n",
    "#     Pickle file for patient initialized\n",
    "    pickleFile = open(\"%s.pickle\" % (patient_id) ,\"wb\")\n",
    "#     Data Array for patient initialized\n",
    "    data_array = []\n",
    "    segmented_ids = []\n",
    "    for dcm in patient_segmented_files:\n",
    "        segmented_ids.append(dcm[-9:-4])\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    Retreiving files of the original patient scans for designated scan\n",
    "    Problems occured:\n",
    "    1- Not all patient directories have single scan history\n",
    "    Patients with such issue are: 06-0138, 19-5960.\n",
    "    Resolved by validating ID for every scan\n",
    "    Status: Resolved by defining an aiding module 'multipleVisits' that operates in the else statement\n",
    "    \n",
    "    if(one visit)-> undergo the process of retreiving patient scans images\n",
    "    else -> use the 'multipleVisits' module to identify the visit date containing scans\n",
    "            segmented followed by retreiving patient data\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    \n",
    "    if(len(os.listdir(original_directory_path+patient_id)) == 1):\n",
    "#         If only one visit for the patient\n",
    "        visit_date = os.listdir(original_directory_path+patient_id)[0]\n",
    "        seg_data_array = []\n",
    "        for scan_directory in os.listdir(original_directory_path+patient_id+\"/\"+visit_date):\n",
    "#             Scan directory is the level in which scan type folders of that visit is reached\n",
    "#             if any(scan_directory[-5:] in segment_scan for segment_scan in patient_segmented_files):\n",
    "            if scan_directory[-5:] in segmented_ids:\n",
    "#                 Handling segmentation data first and then the original\n",
    "                seg_index = 0\n",
    "                for index, name in enumerate(patient_segmented_files):\n",
    "                    if scan_directory[-5:] in name:\n",
    "                        seg_index = index\n",
    "                path_to_seg = segmented_directory_path+patient_id+\"/DSO\"\n",
    "                seg_data_array.append(segmentationArray(path_to_seg, patient_segmented_files[seg_index]))\n",
    "#                 Segmented Data appended to array, now original\n",
    "                folder_containing_scans = scan_directory\n",
    "                path_for_scans = original_directory_path+patient_id+\"/\"+visit_date+\"/\"+folder_containing_scans\n",
    "                scan_array = []\n",
    "                images = os.listdir(path_for_scans)                \n",
    "                ordered_dict_of_filenames = dicomSort(path_for_scans, images)\n",
    "                for image in sorted(ordered_dict_of_filenames.keys()):\n",
    "                    ds = pydicom.dcmread(path_for_scans+\"/\"+ordered_dict_of_filenames[image]).pixel_array\n",
    "                    scan_array.append(ds)\n",
    "                    \n",
    "                data_array.append(scan_array)\n",
    "        data_array.append(seg_data_array)\n",
    "        pickle.dump(data_array,pickleFile)\n",
    "        pickleFile.close()\n",
    "        \n",
    "        \n",
    "#     In case of having multiple visits by the patient\n",
    "    else:\n",
    "        folder_containing_scans = multipleVisits(patient_id, patient_segmented_files)\n",
    "        for folder in folder_containing_scans:\n",
    "            visit_date = folder\n",
    "        seg_data_array = []\n",
    "        for scan_directory in os.listdir(original_directory_path+patient_id+\"/\"+visit_date):\n",
    "#             if any(scan_directory[-5:] in segment_scan for segment_scan in patient_segmented_files):\n",
    "            if scan_directory[-5:] in segmented_ids:\n",
    "                seg_index = 0\n",
    "                for index, name in enumerate(patient_segmented_files):\n",
    "                    if scan_directory[-5:] in name:\n",
    "                        seg_index = index\n",
    "                path_to_seg = segmented_directory_path+patient_id+\"/DSO\"\n",
    "                seg_data_array.append(segmentationArray(path_to_seg, patient_segmented_files[seg_index]))\n",
    "                \n",
    "                \n",
    "                folder_containing_scans = scan_directory\n",
    "                path_for_scans = original_directory_path+patient_id+\"/\"+visit_date+\"/\"+folder_containing_scans\n",
    "                scan_array = []\n",
    "                images = os.listdir(path_for_scans)\n",
    "                ordered_dict_of_filenames = dicomSort(path_for_scans, images)\n",
    "                for image in sorted(ordered_dict_of_filenames.keys()):\n",
    "                    ds = pydicom.dcmread(path_for_scans+\"/\"+ordered_dict_of_filenames[image]).pixel_array\n",
    "                    scan_array.append(ds)\n",
    "                data_array.append(scan_array)\n",
    "        data_array.append(seg_data_array)\n",
    "        pickle.dump(data_array,pickleFile)\n",
    "        pickleFile.close()                \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Completed so far, pickle for gathering all designated original data scans for every segmentation\n",
    "# scan has been completed. Testing for results. Anomalies detected with multiple\n",
    "# with extra number of scans\n",
    "# The final data array contains all the original patient scans and their respective segmentation\n",
    "# The code below tests for one patient. Further confirmation is essential for the model training phase!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DONE\n"
     ]
    }
   ],
   "source": [
    "result_files = os.listdir(result_directory_path)\n",
    "file = 'TCGA-02-0006.pickle'\n",
    "patient = open(\"%s\" % (result_directory_path+file), \"rb\")\n",
    "patient_data = pickle.load(patient)\n",
    "# for i in patient_data:\n",
    "# print(len(patient_data))\n",
    "# for arrays in patient_data[1]:\n",
    "\n",
    "\n",
    "# Plotting commands\n",
    "# plt.figure(figsize=(20,10))\n",
    "# plt.subplot(121)\n",
    "# plt.imshow(np.squeeze(patient_data[1][11]), 'gray')\n",
    "# plt.subplot(122)\n",
    "# plt.imshow(np.squeeze(patient_data[4][1][:,:,11]), 'gray')\n",
    "print (\"DONE\")\n",
    "# for seg_array in patient_data[4]:\n",
    "#     plt.figure(figsize=(20,10))\n",
    "#     plt.imshow(np.squeeze(seg_array[:,:,4]),'gray')\n",
    "#     break\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}