statistics_train_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89a4c56f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/f/241129-zhipu-thyroid-datas/51-trainResults/res_optimizeThyNod1.6_mul_nodules1.5_axpG63_224_20241206T0943_sz224/\n",
      "['imgs_fname_test.txt', 'imgs_fname_train.txt', 'labels.txt', 'linknet18_32_0.5_multi_224', 'train_Thynodules241206t094259.log']\n",
      "len of file:[imgs_fname_test.txt]=3696, first img=thyroidNodules_axp-036.dcm_0001\n",
      "\n",
      "imgs_fname_test.txt contains[13]:\n",
      "thyroidNodules_axp-006: 257, 61\n",
      "thyroidNodules_axp-016: 494, 160\n",
      "thyroidNodules_axp-024: 301, 269\n",
      "thyroidNodules_axp-030: 237, 117\n",
      "thyroidNodules_axp-036: 159, 90\n",
      "thyroidNodules_axp-043: 324, 206\n",
      "thyroidNodules_axp-053: 285, 109\n",
      "thyroidNodules_axp-056: 250, 74\n",
      "thyroidNodules_axp-058: 296, 66\n",
      "thyroidNodules_axp-061: 161, 73\n",
      "thyroidNodules_axp-086: 255, 103\n",
      "thyroidNodules_axp-088: 268, 47\n",
      "thyroidNodules_axp-089: 409, 409\n",
      "len of file:[imgs_fname_train.txt]=16537, first img=thyroidNodules_axp-013.dcm_0001\n",
      "\n",
      "imgs_fname_train.txt contains[50]:\n",
      "thyroidNodules_axp-001: 423, 89\n",
      "thyroidNodules_axp-002: 320, 77\n",
      "thyroidNodules_axp-005: 452, 171\n",
      "thyroidNodules_axp-009: 206, 53\n",
      "thyroidNodules_axp-010: 386, 48\n",
      "thyroidNodules_axp-012: 265, 35\n",
      "thyroidNodules_axp-013: 281, 33\n",
      "thyroidNodules_axp-014: 417, 64\n",
      "thyroidNodules_axp-015: 377, 198\n",
      "thyroidNodules_axp-017: 411, 244\n",
      "thyroidNodules_axp-018: 526, 106\n",
      "thyroidNodules_axp-019: 266, 129\n",
      "thyroidNodules_axp-020: 323, 263\n",
      "thyroidNodules_axp-023: 236, 236\n",
      "thyroidNodules_axp-031: 217, 133\n",
      "thyroidNodules_axp-034: 261, 228\n",
      "thyroidNodules_axp-035: 162, 158\n",
      "thyroidNodules_axp-037: 272, 239\n",
      "thyroidNodules_axp-041: 192, 157\n",
      "thyroidNodules_axp-044: 427, 120\n",
      "thyroidNodules_axp-045: 300, 191\n",
      "thyroidNodules_axp-046: 542, 290\n",
      "thyroidNodules_axp-047: 316, 167\n",
      "thyroidNodules_axp-048: 555, 36\n",
      "thyroidNodules_axp-049: 371, 54\n",
      "thyroidNodules_axp-050: 620, 285\n",
      "thyroidNodules_axp-051: 247, 81\n",
      "thyroidNodules_axp-052: 377, 171\n",
      "thyroidNodules_axp-054: 296, 95\n",
      "thyroidNodules_axp-055: 482, 421\n",
      "thyroidNodules_axp-062: 277, 134\n",
      "thyroidNodules_axp-063: 193, 78\n",
      "thyroidNodules_axp-064: 199, 96\n",
      "thyroidNodules_axp-065: 235, 149\n",
      "thyroidNodules_axp-068: 224, 116\n",
      "thyroidNodules_axp-069: 360, 76\n",
      "thyroidNodules_axp-071: 307, 34\n",
      "thyroidNodules_axp-072: 285, 39\n",
      "thyroidNodules_axp-073: 257, 31\n",
      "thyroidNodules_axp-074: 383, 268\n",
      "thyroidNodules_axp-075: 312, 76\n",
      "thyroidNodules_axp-076: 317, 101\n",
      "thyroidNodules_axp-077: 393, 164\n",
      "thyroidNodules_axp-078: 341, 105\n",
      "thyroidNodules_axp-079: 428, 93\n",
      "thyroidNodules_axp-081: 272, 128\n",
      "thyroidNodules_axp-082: 517, 20\n",
      "thyroidNodules_axp-084: 265, 138\n",
      "thyroidNodules_axp-087: 222, 62\n",
      "thyroidNodules_axp-101: 224, 190\n"
     ]
    }
   ],
   "source": [
    "# statistics the data features about train and test.\n",
    "# eton@241207\n",
    "\n",
    "import os\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "\n",
    "trainResFolderParent=r'/mnt/f/241129-zhipu-thyroid-datas/51-trainResults'\n",
    "resFolder=r'res_optimizeThyNod1.6_mul_nodules1.5_axpG63_224_20241206T0943_sz224/'\n",
    "\n",
    "\n",
    "\n",
    "trainDataListfName='imgs_fname_train.txt'\n",
    "validateDataListfName='imgs_fname_test.txt'\n",
    "\n",
    "datasetTypes=[validateDataListfName, trainDataListfName]\n",
    "\n",
    "#thyroidNodules_axp-013.dcm_0001\n",
    "dcmlistSuffix=\".dcm\"\n",
    "dcmfolderSuffix=dcmlistSuffix+\"_frms\" \n",
    "\n",
    "\n",
    "# List of image file extensions\n",
    "image_extensions = ['*.jpg', '*.jpeg', '*.png', '*.gif', '*.bmp', '*.tiff']\n",
    "json_extension=\"*.json\"\n",
    "dataActualFolder=r'/mnt/f/241129-zhipu-thyroid-datas/31-labelmeFormatOrganized/241207_all82AixThyroidNodules'\n",
    "\n",
    "def statis_emptyJson_and_total(onecasename:str):\n",
    "    onedcmfolder=onecasename+dcmfolderSuffix\n",
    "    dcmpath=os.path.join(dataActualFolder, onedcmfolder)\n",
    "    if not os.path.isdir(dcmpath):\n",
    "        print(f\"Err:file:{onedcmfolder} not exist, ignore {dcmpath}\")\n",
    "        return\n",
    "    files_in_dir = os.listdir(dcmpath)\n",
    "    \n",
    "    if len(files_in_dir) < 2:\n",
    "        print(f\"Empty dirctory : {dcmpath}\")\n",
    "        return\n",
    "    # Define the folder path\n",
    "    folder_path = Path(dcmpath)\n",
    "    # List to store image file paths\n",
    "    image_files = []\n",
    "    json_files=[]\n",
    "\n",
    "    # Loop through each extension and find matching files\n",
    "    for extension in image_extensions:\n",
    "        image_files.extend(folder_path.glob(extension))\n",
    "    json_files.extend(folder_path.glob(json_extension))\n",
    "\n",
    "    lmstartJsonfile=[ii for ii in json_files if \"lmstart.json\" in str(ii)]\n",
    "    if len(lmstartJsonfile)>0:\n",
    "        json_files.remove(lmstartJsonfile[0])\n",
    "\n",
    "    #print(f\"debug: images count:{len(image_files)}, json count{len(json_files)}\")\n",
    "    return (len(image_files), len(json_files))\n",
    "\n",
    "def processImglistLines(imgnamelines:list):\n",
    "    dcmnameset=set()\n",
    "    dcmname=None\n",
    "    for imgline in imgnamelines:\n",
    "        nameParts=imgline.split(dcmlistSuffix)\n",
    "        if type(nameParts) is list and len(nameParts)>1:\n",
    "            if len(nameParts[0]) > 2:\n",
    "                dcmname=nameParts[0]\n",
    "        if type(dcmname) is not None:\n",
    "            dcmnameset.add(dcmname)\n",
    "    #print(imgnameline.split(\".dcm\"))\n",
    "    sortedList=sorted(dcmnameset)\n",
    "    #print(f\"debug: dcm name set:{type(sortedList)}\")\n",
    "    return sortedList\n",
    "\n",
    "def readImgListFile():\n",
    "    iOneResfolder=os.path.join(trainResFolderParent, resFolder)\n",
    "    print(iOneResfolder)\n",
    "\n",
    "    dirItems = sorted(os.listdir(iOneResfolder))\n",
    "    print(dirItems)\n",
    "    \n",
    "    for iitem in dirItems:\n",
    "        statis_list=[]\n",
    "        if iitem in datasetTypes:\n",
    "            imglists=None\n",
    "            \n",
    "            iitemPath=os.path.join(iOneResfolder, iitem)\n",
    "            if not os.path.isfile(iitemPath):\n",
    "                print(f\"Err:file:{iitem} not exist, ignore\")\n",
    "            with open(iitemPath, 'r') as fp:\n",
    "                imglists=fp.readlines()\n",
    "            if type(imglists) == None:\n",
    "                print(f\"Err: read from file[{iitem}] failed.!\")\n",
    "                return\n",
    "            print(f\"len of file:[{iitem}]={len(imglists)}, first img={imglists[0] if len(imglists)>0 else imglists}\")\n",
    "            dcmNameset=processImglistLines(imglists)\n",
    "            print(f\"{iitem} contains[{len(dcmNameset)}]:\")\n",
    "            for dcmname in dcmNameset:\n",
    "                imgCnt, jsonCnt=statis_emptyJson_and_total(dcmname)\n",
    "                print(f\"{dcmname}: {imgCnt}, {jsonCnt}\") \n",
    "                statis_list.append([dcmname, imgCnt, jsonCnt])\n",
    "            \n",
    "            statis_list.insert(0, [\"caseName\",\"ImgCnt\", \"AnnoCnt\"])\n",
    "            df = pd.DataFrame(statis_list)\n",
    "            \n",
    "            df.to_csv(f\"{iitem}.csv\", mode='a', header=True, index=False)\n",
    "            \n",
    "readImgListFile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5edf4890",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "debug: images count:423, json count90\n"
     ]
    }
   ],
   "source": [
    "\n",
    "    \n",
    "dcmdirname=r'thyroidNodules_axp-001'\n",
    "statis_emptyJson_and_total(dcmdirname)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ffa0f8c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                   Version\n",
      "------------------------- --------------\n",
      "anyio                     4.7.0\n",
      "argon2-cffi               23.1.0\n",
      "argon2-cffi-bindings      21.2.0\n",
      "arrow                     1.3.0\n",
      "asttokens                 3.0.0\n",
      "async-lru                 2.0.4\n",
      "attrs                     24.2.0\n",
      "babel                     2.16.0\n",
      "beautifulsoup4            4.12.3\n",
      "bleach                    6.2.0\n",
      "certifi                   2024.8.30\n",
      "cffi                      1.17.1\n",
      "charset-normalizer        3.4.0\n",
      "comm                      0.2.2\n",
      "debugpy                   1.8.9\n",
      "decorator                 5.1.1\n",
      "defusedxml                0.7.1\n",
      "exceptiongroup            1.2.2\n",
      "executing                 2.1.0\n",
      "fastjsonschema            2.21.1\n",
      "fqdn                      1.5.1\n",
      "h11                       0.14.0\n",
      "httpcore                  1.0.7\n",
      "httpx                     0.28.1\n",
      "idna                      3.10\n",
      "ipykernel                 6.29.5\n",
      "ipython                   8.30.0\n",
      "ipywidgets                8.1.5\n",
      "isoduration               20.11.0\n",
      "jedi                      0.19.2\n",
      "Jinja2                    3.1.4\n",
      "json5                     0.10.0\n",
      "jsonpointer               3.0.0\n",
      "jsonschema                4.23.0\n",
      "jsonschema-specifications 2024.10.1\n",
      "jupyter                   1.1.1\n",
      "jupyter_client            8.6.3\n",
      "jupyter-console           6.6.3\n",
      "jupyter_core              5.7.2\n",
      "jupyter-events            0.10.0\n",
      "jupyter-lsp               2.2.5\n",
      "jupyter_server            2.14.2\n",
      "jupyter_server_terminals  0.5.3\n",
      "jupyterlab                4.3.2\n",
      "jupyterlab_pygments       0.3.0\n",
      "jupyterlab_server         2.27.3\n",
      "jupyterlab_widgets        3.0.13\n",
      "MarkupSafe                3.0.2\n",
      "matplotlib-inline         0.1.7\n",
      "mistune                   3.0.2\n",
      "nbclient                  0.10.1\n",
      "nbconvert                 7.16.4\n",
      "nbformat                  5.10.4\n",
      "nest-asyncio              1.6.0\n",
      "notebook                  7.3.1\n",
      "notebook_shim             0.2.4\n",
      "numpy                     2.1.3\n",
      "overrides                 7.7.0\n",
      "packaging                 24.2\n",
      "pandas                    2.2.3\n",
      "pandocfilters             1.5.1\n",
      "parso                     0.8.4\n",
      "pexpect                   4.9.0\n",
      "pip                       24.3.1\n",
      "platformdirs              4.3.6\n",
      "prometheus_client         0.21.1\n",
      "prompt_toolkit            3.0.48\n",
      "psutil                    6.1.0\n",
      "ptyprocess                0.7.0\n",
      "pure_eval                 0.2.3\n",
      "pycparser                 2.22\n",
      "Pygments                  2.18.0\n",
      "python-dateutil           2.9.0.post0\n",
      "python-json-logger        2.0.7\n",
      "pytz                      2024.2\n",
      "PyYAML                    6.0.2\n",
      "pyzmq                     26.2.0\n",
      "referencing               0.35.1\n",
      "requests                  2.32.3\n",
      "rfc3339-validator         0.1.4\n",
      "rfc3986-validator         0.1.1\n",
      "rpds-py                   0.22.3\n",
      "Send2Trash                1.8.3\n",
      "setuptools                75.5.0\n",
      "six                       1.17.0\n",
      "sniffio                   1.3.1\n",
      "soupsieve                 2.6\n",
      "stack-data                0.6.3\n",
      "terminado                 0.18.1\n",
      "tinycss2                  1.4.0\n",
      "tomli                     2.2.1\n",
      "tornado                   6.4.2\n",
      "traitlets                 5.14.3\n",
      "types-python-dateutil     2.9.0.20241206\n",
      "typing_extensions         4.12.2\n",
      "tzdata                    2024.2\n",
      "uri-template              1.3.0\n",
      "urllib3                   2.2.3\n",
      "wcwidth                   0.2.13\n",
      "webcolors                 24.11.1\n",
      "webencodings              0.5.1\n",
      "websocket-client          1.8.0\n",
      "wheel                     0.45.1\n",
      "widgetsnbextension        4.0.13\n",
      "yolk3k                    0.9\n",
      "/mnt/d/000-srcs/210822-thyroid_train\n"
     ]
    }
   ],
   "source": [
    "!pip list\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cc3fc4a1",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "C extension: None not built. If you want to import pandas from the source directory, you may need to run 'python setup.py build_ext' to build the C extensions first.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;31m# numpy compat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m     from pandas.compat import (\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0mis_numpy_dev\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_is_numpy_dev\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# pyright: ignore[reportUnusedImport] # noqa: F401\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/compat/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompressors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mis_numpy_dev\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m from pandas.compat.pyarrow import (\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/compat/numpy/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m_nlv\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mVersion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_min_numpy_ver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     raise ImportError(\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0;34mf\"this version of pandas is incompatible with numpy < {_min_numpy_ver}\\n\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: this version of pandas is incompatible with numpy < 1.22.4\nyour numpy version is 1.21.5.\nPlease upgrade numpy to >= 1.22.4 to use this pandas version",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_1981/3648170110.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mImportError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_err\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pragma: no cover\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0m_module\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_err\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m     raise ImportError(\n\u001b[0m\u001b[1;32m     32\u001b[0m         \u001b[0;34mf\"C extension: {_module} not built. If you want to import \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;34m\"pandas from the source directory, you may need to run \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: C extension: None not built. If you want to import pandas from the source directory, you may need to run 'python setup.py build_ext' to build the C extensions first."
     ]
    }
   ],
   "source": [
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f480fd0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}