src/synthetic_data/generate_synthetic_data_seqFISH.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d2cee82-d84e-4c57-af58-6ce961a3f819",
   "metadata": {},
   "source": [
    "To generate synthetic VisiumHD data from seqFISH+, please read and run all the cells below. Thanks!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9fcd48a-2f55-43b4-befd-8d646ea634cf",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Install prerequisite libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7453e3e3-a55c-47fb-ab83-2c3743833b89",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: pip in /opt/conda/lib/python3.10/site-packages (24.2)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: scipy in /home/oneai/.local/lib/python3.10/site-packages (1.10.0)\n",
      "Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /home/oneai/.local/lib/python3.10/site-packages (from scipy) (1.22.4)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: shapely in /home/oneai/.local/lib/python3.10/site-packages (2.0.0)\n",
      "Requirement already satisfied: numpy>=1.14 in /home/oneai/.local/lib/python3.10/site-packages (from shapely) (1.22.4)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: tifffile in /home/oneai/.local/lib/python3.10/site-packages (2022.10.10)\n",
      "Requirement already satisfied: numpy>=1.19.2 in /home/oneai/.local/lib/python3.10/site-packages (from tifffile) (1.22.4)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: plotly in /home/oneai/.local/lib/python3.10/site-packages (5.13.1)\n",
      "Requirement already satisfied: tenacity>=6.2.0 in /home/oneai/.local/lib/python3.10/site-packages (from plotly) (9.0.0)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: tensorflow-gpu==2.10.0 in /opt/conda/lib/python3.10/site-packages (2.10.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (2.1.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=2.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (24.3.25)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (0.4.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (0.2.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (1.66.1)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (3.11.0)\n",
      "Requirement already satisfied: keras<2.11,>=2.10.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (2.10.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (1.1.2)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (18.1.1)\n",
      "Requirement already satisfied: numpy>=1.20 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (1.22.4)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (3.3.0)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (24.0)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (3.19.6)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (69.5.1)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (1.16.0)\n",
      "Requirement already satisfied: tensorboard<2.11,>=2.10 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (2.10.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (0.37.1)\n",
      "Requirement already satisfied: tensorflow-estimator<2.11,>=2.10.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (2.10.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (2.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (4.12.2)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /home/oneai/.local/lib/python3.10/site-packages (from tensorflow-gpu==2.10.0) (1.14.1)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.10/site-packages (from astunparse>=1.6.0->tensorflow-gpu==2.10.0) (0.43.0)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (2.33.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (0.4.6)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/oneai/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (3.7)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (0.6.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (1.8.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /home/oneai/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (3.0.4)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (5.4.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (0.4.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (4.9)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (2.0.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (1.26.19)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (2024.7.4)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.10/site-packages (from werkzeug>=1.0.1->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (2.1.5)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/conda/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (0.6.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow-gpu==2.10.0) (3.2.2)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: stardist in /home/oneai/.local/lib/python3.10/site-packages (0.9.1)\n",
      "Requirement already satisfied: csbdeep>=0.8.0 in /home/oneai/.local/lib/python3.10/site-packages (from stardist) (0.8.0)\n",
      "Requirement already satisfied: scikit-image in /home/oneai/.local/lib/python3.10/site-packages (from stardist) (0.19.3)\n",
      "Requirement already satisfied: numba in /home/oneai/.local/lib/python3.10/site-packages (from stardist) (0.55.2)\n",
      "Requirement already satisfied: imageio in /home/oneai/.local/lib/python3.10/site-packages (from stardist) (2.35.1)\n",
      "Requirement already satisfied: numpy in /home/oneai/.local/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (1.22.4)\n",
      "Requirement already satisfied: scipy in /home/oneai/.local/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (1.10.0)\n",
      "Requirement already satisfied: matplotlib in /home/oneai/.local/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (3.6.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (1.16.0)\n",
      "Requirement already satisfied: tifffile in /home/oneai/.local/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (2022.10.10)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (4.66.2)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from csbdeep>=0.8.0->stardist) (24.0)\n",
      "Requirement already satisfied: pillow>=8.3.2 in /home/oneai/.local/lib/python3.10/site-packages (from imageio->stardist) (10.4.0)\n",
      "Requirement already satisfied: llvmlite<0.39,>=0.38.0rc1 in /home/oneai/.local/lib/python3.10/site-packages (from numba->stardist) (0.38.1)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from numba->stardist) (69.5.1)\n",
      "Requirement already satisfied: networkx>=2.2 in /home/oneai/.local/lib/python3.10/site-packages (from scikit-image->stardist) (3.3)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in /home/oneai/.local/lib/python3.10/site-packages (from scikit-image->stardist) (1.6.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib->csbdeep>=0.8.0->stardist) (1.2.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib->csbdeep>=0.8.0->stardist) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib->csbdeep>=0.8.0->stardist) (4.53.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib->csbdeep>=0.8.0->stardist) (1.4.5)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib->csbdeep>=0.8.0->stardist) (3.1.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib->csbdeep>=0.8.0->stardist) (2.9.0)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: geopandas in /home/oneai/.local/lib/python3.10/site-packages (0.12.2)\n",
      "Requirement already satisfied: pandas>=1.0.0 in /home/oneai/.local/lib/python3.10/site-packages (from geopandas) (1.5.2)\n",
      "Requirement already satisfied: shapely>=1.7 in /home/oneai/.local/lib/python3.10/site-packages (from geopandas) (2.0.0)\n",
      "Requirement already satisfied: fiona>=1.8 in /home/oneai/.local/lib/python3.10/site-packages (from geopandas) (1.9.6)\n",
      "Requirement already satisfied: pyproj>=2.6.1.post1 in /home/oneai/.local/lib/python3.10/site-packages (from geopandas) (3.6.1)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from geopandas) (24.0)\n",
      "Requirement already satisfied: attrs>=19.2.0 in /opt/conda/lib/python3.10/site-packages (from fiona>=1.8->geopandas) (24.2.0)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.10/site-packages (from fiona>=1.8->geopandas) (2024.7.4)\n",
      "Requirement already satisfied: click~=8.0 in /home/oneai/.local/lib/python3.10/site-packages (from fiona>=1.8->geopandas) (8.1.7)\n",
      "Requirement already satisfied: click-plugins>=1.0 in /home/oneai/.local/lib/python3.10/site-packages (from fiona>=1.8->geopandas) (1.1.1)\n",
      "Requirement already satisfied: cligj>=0.5 in /home/oneai/.local/lib/python3.10/site-packages (from fiona>=1.8->geopandas) (0.7.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from fiona>=1.8->geopandas) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas>=1.0.0->geopandas) (2.9.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/oneai/.local/lib/python3.10/site-packages (from pandas>=1.0.0->geopandas) (2022.7.1)\n",
      "Requirement already satisfied: numpy>=1.21.0 in /home/oneai/.local/lib/python3.10/site-packages (from pandas>=1.0.0->geopandas) (1.22.4)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: scanpy in /home/oneai/.local/lib/python3.10/site-packages (1.9.1)\n",
      "Requirement already satisfied: anndata>=0.7.4 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (0.8.0)\n",
      "Requirement already satisfied: numpy>=1.17.0 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (1.22.4)\n",
      "Requirement already satisfied: matplotlib>=3.4 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (3.6.2)\n",
      "Requirement already satisfied: pandas>=1.0 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (1.5.2)\n",
      "Requirement already satisfied: scipy>=1.4 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (1.10.0)\n",
      "Requirement already satisfied: seaborn in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (0.13.2)\n",
      "Requirement already satisfied: h5py>=3 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (3.11.0)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from scanpy) (4.66.2)\n",
      "Requirement already satisfied: scikit-learn>=0.22 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (1.2.0)\n",
      "Requirement already satisfied: statsmodels>=0.10.0rc2 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (0.14.2)\n",
      "Requirement already satisfied: patsy in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (0.5.6)\n",
      "Requirement already satisfied: networkx>=2.3 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (3.3)\n",
      "Requirement already satisfied: natsort in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (8.4.0)\n",
      "Requirement already satisfied: joblib in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (1.4.2)\n",
      "Requirement already satisfied: numba>=0.41.0 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (0.55.2)\n",
      "Requirement already satisfied: umap-learn>=0.3.10 in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (0.5.6)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from scanpy) (24.0)\n",
      "Requirement already satisfied: session-info in /home/oneai/.local/lib/python3.10/site-packages (from scanpy) (1.0.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (1.2.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (4.53.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (1.4.5)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (10.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/oneai/.local/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (3.1.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=3.4->scanpy) (2.9.0)\n",
      "Requirement already satisfied: llvmlite<0.39,>=0.38.0rc1 in /home/oneai/.local/lib/python3.10/site-packages (from numba>=0.41.0->scanpy) (0.38.1)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from numba>=0.41.0->scanpy) (69.5.1)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/oneai/.local/lib/python3.10/site-packages (from pandas>=1.0->scanpy) (2022.7.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/oneai/.local/lib/python3.10/site-packages (from scikit-learn>=0.22->scanpy) (3.1.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from patsy->scanpy) (1.16.0)\n",
      "Requirement already satisfied: pynndescent>=0.5 in /home/oneai/.local/lib/python3.10/site-packages (from umap-learn>=0.3.10->scanpy) (0.5.13)\n",
      "Requirement already satisfied: stdlib-list in /home/oneai/.local/lib/python3.10/site-packages (from session-info->scanpy) (0.10.0)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: fastparquet in /home/oneai/.local/lib/python3.10/site-packages (2024.5.0)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /home/oneai/.local/lib/python3.10/site-packages (from fastparquet) (1.5.2)\n",
      "Requirement already satisfied: numpy in /home/oneai/.local/lib/python3.10/site-packages (from fastparquet) (1.22.4)\n",
      "Requirement already satisfied: cramjam>=2.3 in /home/oneai/.local/lib/python3.10/site-packages (from fastparquet) (2.8.3)\n",
      "Requirement already satisfied: fsspec in /home/oneai/.local/lib/python3.10/site-packages (from fastparquet) (2024.6.1)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from fastparquet) (24.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas>=1.5.0->fastparquet) (2.9.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/oneai/.local/lib/python3.10/site-packages (from pandas>=1.5.0->fastparquet) (2022.7.1)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: imagecodecs in /home/oneai/.local/lib/python3.10/site-packages (2024.6.1)\n",
      "Requirement already satisfied: numpy in /home/oneai/.local/lib/python3.10/site-packages (from imagecodecs) (1.22.4)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: zarr in /home/oneai/.local/lib/python3.10/site-packages (2.17.1)\n",
      "Requirement already satisfied: asciitree in /home/oneai/.local/lib/python3.10/site-packages (from zarr) (0.3.3)\n",
      "Requirement already satisfied: numpy>=1.21.1 in /home/oneai/.local/lib/python3.10/site-packages (from zarr) (1.22.4)\n",
      "Requirement already satisfied: numcodecs>=0.10.0 in /home/oneai/.local/lib/python3.10/site-packages (from zarr) (0.13.0)\n",
      "Requirement already satisfied: fasteners in /home/oneai/.local/lib/python3.10/site-packages (from zarr) (0.19)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: scipy in /home/oneai/.local/lib/python3.10/site-packages (1.10.0)\n",
      "Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /home/oneai/.local/lib/python3.10/site-packages (from scipy) (1.22.4)\n",
      "Looking in indexes: https://jfrog-proxy.services.p171649450587.aws-emea.sanofi.com/artifactory/api/pypi/pypi-one_ai-virtual/simple, https://pypi.org/simple\n",
      "Requirement already satisfied: h5py in /home/oneai/.local/lib/python3.10/site-packages (3.11.0)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /home/oneai/.local/lib/python3.10/site-packages (from h5py) (1.22.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade pip\n",
    "!pip install scipy\n",
    "!pip install shapely\n",
    "!pip install tifffile\n",
    "!pip install plotly\n",
    "!pip install tensorflow-gpu==2.10.0\n",
    "!pip install stardist\n",
    "!pip install geopandas\n",
    "!pip install scanpy\n",
    "!pip install fastparquet\n",
    "!pip install imagecodecs\n",
    "!pip install zarr\n",
    "!pip install scipy\n",
    "!pip install h5py"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f79fb2c-0fd9-4bd4-8be9-4d1bd04d8733",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Import Relevant Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16e4dc02-2b8d-4e00-9cbd-8a4d151ca5af",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tifffile as tifi # Package to read the WSI (whole slide image)\n",
    "from csbdeep.utils import normalize # Image normalization\n",
    "from shapely.geometry import Polygon, Point # Representing bins and cells as Shapely Polygons and Point objects\n",
    "from shapely import wkt\n",
    "import geopandas as gpd # Geopandas for storing Shapely objects\n",
    "from matplotlib.colors import ListedColormap\n",
    "import matplotlib.pyplot as plt\n",
    "import scanpy as sc\n",
    "import pandas as pd\n",
    "from scipy import sparse\n",
    "import anndata\n",
    "import os\n",
    "import gzip\n",
    "import numpy as np\n",
    "import re\n",
    "import shapely\n",
    "import zarr\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a91a092e-781d-4a9e-8777-d3bb9c99309c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Create folders to store synthetic data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37e30a8b-77f8-4d2c-97c2-8274eb0d23a3",
   "metadata": {},
   "source": [
    "For both the `seqfish_dir` and `enact_data_dir`, change `\"/home/oneai/\"` to the directory that stores this repo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01f77ecd-3f9a-4a39-bbb2-e90e851ec360",
   "metadata": {},
   "outputs": [],
   "source": [
    "seqfish_dir = \"/home/oneai/oneai-dda-spatialtr-visiumhd_analysis/synthetic_data/seqFISH\" # Update it to the directory where you want to save the synthetic data\n",
    "enact_data_dir = \"/home/oneai/oneai-dda-spatialtr-visiumhd_analysis/cache/seqfish/chunks\" # Directory that saves all the input and results of the enact pipeline, \n",
    "# should end with \"oneai-dda-spatialtr-visiumhd_analysis/cache/seqfish/chunks\"\n",
    "\n",
    "transcripts_df_chunks_dir = os.path.join(seqfish_dir, \"transcripts_patches\") # Directory to store the files that contain the transcripts info for each chunk\n",
    "output_dir = os.path.join(enact_data_dir, \"bins_gdf\") # Directory to store the generated synthetic binned transcript counts\n",
    "cells_df_chunks_dir =  os.path.join(enact_data_dir,\"cells_gdf\") # Directory to store the generated synthetic binned transcript counts\n",
    "\n",
    "# Making relevant directories\n",
    "os.makedirs(seqfish_dir, exist_ok=True)\n",
    "os.makedirs(enact_data_dir, exist_ok=True)\n",
    "os.makedirs(transcripts_df_chunks_dir, exist_ok=True)\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "os.makedirs(cells_df_chunks_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0048c41f-18ee-4b92-b7ea-680956330667",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Download seqFISH+ data\n",
    "\n",
    "1. Download \"ROIs_Experiment1_NIH3T3.zip\" from https://zenodo.org/records/2669683#.Xqi1w5NKg6g to seqfish_dir. The zipfile contains cell segmentation files\n",
    "2. Download \"run1.csv.gz\" from https://github.com/MonashBioinformaticsPlatform/seqfish-hack. It contains the tidy format of \"seqFISH+_NIH3T3_point_locations.zip\" from the official seqFISH+ zenodo site"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46a8d90a-65dd-4e93-b4e2-4a257d6e1dc7",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Load Cell & Transcripts Info"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a40feb4c-1510-4222-bdec-a5e419758f32",
   "metadata": {},
   "source": [
    "This following cells first unzip \"ROIs_Experiment1_NIH3T3.zip\" to extract the cell segmentation information. Then load transcripts dataframe from \"run1.csv.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7bb6152-3999-4ccf-9a08-8fad268ab972",
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "import os\n",
    "zip_file_path = os.path.join(seqfish_dir, \"ROIs_Experiment1_NIH3T3.zip\")\n",
    "\n",
    "# Open the ZIP file and extract all the contents\n",
    "with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
    "    zip_ref.extractall(seqfish_dir)\n",
    "\n",
    "print(f'Files extracted to {seqfish_dir}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "062ee054-6e0e-4c2b-9782-9e2b328c18e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path =  os.path.join(seqfish_dir, \"run1.csv.gz\")\n",
    "\n",
    "transcripts_df = pd.read_csv(file_path, compression='gzip')\n",
    "print(transcripts_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb2be572-8903-4539-8306-087cf61aa82d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert from pixel to um\n",
    "transcripts_df.x = transcripts_df.x*0.103\n",
    "transcripts_df.y = transcripts_df.y*0.103\n",
    "# label cell to include fov and cell number\n",
    "transcripts_df['new_cell_name'] = transcripts_df.apply(lambda x: f\"{x['fov']}_Cell_{x['cell']}\", axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c8134a2-9a2b-41b6-81ab-4e292609e2f2",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Generate Ground Truth"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ae7d39d-f065-454a-bea6-7d31f57139fd",
   "metadata": {},
   "source": [
    "The following cell will generate and save the ground truth of the synthetic VisiumHD data for the use of bin-to-cell assignment methods evaluation. Ground truth dataframe consists of rows representing the transcript counts of each cell. Each column represents a gene feature (gene feature name is also the column name)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9bc5483-2357-40aa-a5b8-1a140b08967a",
   "metadata": {},
   "outputs": [],
   "source": [
    "groundtruth_df = transcripts_df.pivot_table(index=['new_cell_name'], columns='gene', aggfunc='size', fill_value=0)\n",
    "ground_truth_file = os.path.join(seqfish_dir, \"groundtruth.csv\")\n",
    "groundtruth_df.to_csv(ground_truth_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dafe70a1-ed23-4cb6-a7b6-d35e4c01f895",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Generate Synthetic VesiumHD Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bdd8461-7bcc-4101-b26b-765daf975916",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Break transcripts df to patches (based on fov)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0d353b0-74cc-44f1-9a5b-ab5533d5d76a",
   "metadata": {},
   "source": [
    "Break transcripts df to patches based on their field of view (fov), since cell segmentation is done on each individual fov seperately."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60fb886a-5893-40ba-b187-650d6cfb4ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a df for each fov\n",
    "grouped = transcripts_df.groupby(['fov'])\n",
    "for fov, group in grouped:\n",
    "    filename = f\"patch_{fov}.csv\"\n",
    "    output_loc = os.path.join(transcripts_df_chunks_dir, filename)\n",
    "    group.to_csv(output_loc)\n",
    "\n",
    "    print(f\"Saved {filename}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7bbc9ec-675b-4b25-8448-334ed317798a",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Generate synthetic vesiumHD for each patch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99052790-7e12-4851-b9a4-e9ead3a55d0f",
   "metadata": {},
   "source": [
    "Each fov is broken into bins of size 2um x 2um. The synthetic data contains transcript counts orgnized by bin_id. Each row contains transcript counts for a unique bin. Bins with no transcript counts is not included. \n",
    "\n",
    "In addition to all the gene features, there are two additional columns represent the row number and column number of the bin, and a column contains the Shapely polygon item that represents the bin. The first column is the bin_id."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d19155a0-5646-49bd-915c-94737e251bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_synthetic_VesiumHD_data(transcripts_df, bin_size=2):\n",
    "    \n",
    "    filtered_df = transcripts_df.copy()\n",
    "    \n",
    "    # assigne bin to each transcript\n",
    "    filtered_df.loc[:, 'row'] =np.ceil(filtered_df['y'] / bin_size).astype(int)\n",
    "    filtered_df.loc[:, 'column'] = np.ceil(filtered_df['x'] / bin_size).astype(int)\n",
    "    filtered_df.loc[:, 'assigned_bin_id'] = filtered_df.apply(\n",
    "        lambda row: f\"{bin_size}um_\" + str(row['row']).zfill(5) +\"_\"+ str(row['column']).zfill(5),\n",
    "        axis=1)\n",
    "    bin_coordinates = filtered_df[['assigned_bin_id', 'row', 'column']].drop_duplicates().set_index('assigned_bin_id')\n",
    "    bin_gene_matrix = filtered_df.groupby(['assigned_bin_id', 'gene']).size().unstack(fill_value=0)\n",
    "    bin_gene_matrix_with_coords = bin_gene_matrix.merge(bin_coordinates, left_index=True, right_index=True)\n",
    "    return bin_gene_matrix_with_coords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd804c49-dc85-4fa9-85d4-a621cf0598ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract row and column number from the bin_id\n",
    "def extract_numbers(entry):\n",
    "    match = re.search(r'_(\\d{5})_(\\d{5})', entry)\n",
    "    if match:\n",
    "        number1 = int(match.group(1).lstrip('0'))  \n",
    "        number2 = int(match.group(2).lstrip('0'))  \n",
    "        return number2*2-1, number1*2-1\n",
    "    else:\n",
    "        return None, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee921e47-70e4-4bee-92e3-6ce40a0fb50d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "def generate_bin_polys(bins_df, x_col, y_col, bin_size):\n",
    "        \"\"\"Represents the bins as Shapely polygons\n",
    "\n",
    "        Args:\n",
    "            bins_df (pd.DataFrame): bins dataframe\n",
    "            x_col (str): column with the bin centre x-coordinate\n",
    "            y_col (str): column with the bin centre y-coordinate\n",
    "            bin_size (int): bin size in pixels\n",
    "\n",
    "        Returns:\n",
    "            list: list of Shapely polygons\n",
    "        \"\"\"\n",
    "        geometry = []\n",
    "        # Generates Shapely polygons to represent each bin\n",
    "\n",
    "        if True:\n",
    "            half_bin_size = bin_size / 2\n",
    "            bbox_coords = pd.DataFrame(\n",
    "                {\n",
    "                    \"min_x\": bins_df[x_col] - half_bin_size,\n",
    "                    \"min_y\": bins_df[y_col] - half_bin_size,\n",
    "                    \"max_x\": bins_df[x_col] + half_bin_size,\n",
    "                    \"max_y\": bins_df[y_col] + half_bin_size,\n",
    "                }\n",
    "            )\n",
    "            geometry = [\n",
    "                shapely.geometry.box(min_x, min_y, max_x, max_y)\n",
    "                for min_x, min_y, max_x, max_y in tqdm(\n",
    "                    zip(\n",
    "                        bbox_coords[\"min_x\"],\n",
    "                        bbox_coords[\"min_y\"],\n",
    "                        bbox_coords[\"max_x\"],\n",
    "                        bbox_coords[\"max_y\"],\n",
    "                    ),\n",
    "                    total=len(bins_df),\n",
    "                )\n",
    "            ]\n",
    "\n",
    "        return geometry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f1c4071-ff50-4ec1-bd0d-37c8ddecaa54",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loop through all the transcripra_df patches and generate gene-to-bin assignments \n",
    "bin_size = 2\n",
    "transcripts_df_chunks = os.listdir(transcripts_df_chunks_dir)\n",
    "for chunk_fname in transcripts_df_chunks:\n",
    "    output_loc = os.path.join(output_dir, chunk_fname)\n",
    "    if chunk_fname in [\".ipynb_checkpoints\"]:\n",
    "        continue\n",
    "    # if os.path.exists(output_loc):\n",
    "    #     continue\n",
    "    transcripts_df_chunk = pd.read_csv(os.path.join(transcripts_df_chunks_dir, chunk_fname))\n",
    "    bin_df_chunk = generate_synthetic_VesiumHD_data(transcripts_df_chunk, bin_size)\n",
    "    bin_df_chunk['column'] = bin_df_chunk['column']*2-1\n",
    "    bin_df_chunk['row'] = bin_df_chunk['row']*2-1\n",
    "    bin_df_chunk['geometry'] = generate_bin_polys(bin_df_chunk, 'column', 'row', 2)\n",
    "    bin_gdf_chunk = gpd.GeoDataFrame( bin_df_chunk, geometry = bin_df_chunk['geometry'])\n",
    "    bin_gdf_chunk.to_csv(output_loc)\n",
    "   \n",
    "    print(f\"Successfully assigned transcripts to bins for {chunk_fname}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ae8aa8e-0a17-48ae-86ed-81a04ec203dc",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Generate ENACT pipeline cell segmentation input"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2fc8e57-23d4-4f71-971b-6e4e1d9f0267",
   "metadata": {},
   "source": [
    "This session generate the cell_df patches required to run the enact pipeline. The main purpose is to create Shapely polygons that represent the cell outline."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57c34d0c-029c-482f-bc27-fc39e52adf4a",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Load cell boundary data and create cell polygons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b140bc6d-f120-4d18-b302-844bb3b79a63",
   "metadata": {},
   "outputs": [],
   "source": [
    "import read_roi\n",
    "def process_roi_file(key, roi_file_path):\n",
    "    roi_data = read_roi.read_roi_file(roi_file_path)\n",
    "    data = roi_data[key]\n",
    "    # Apply the scaling factor to each coordinate separately\n",
    "    scaled_x = [x * 0.103 for x in data['x']]\n",
    "    scaled_y = [y * 0.103 for y in data['y']]\n",
    "    # Create the list of points using zip on the scaled coordinates\n",
    "    points = [(x, y) for x, y in zip(scaled_x, scaled_y)]\n",
    "    # Create and return the polygon\n",
    "    polygon = Polygon(points)\n",
    "    return polygon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5295212-8548-44a1-b15f-1234bdf28b88",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_fov_from_string(s):\n",
    "    # Search for one or more digits in the string\n",
    "    match = re.search(r'\\d+', s)\n",
    "    if match:\n",
    "        return int(match.group(0))+1 # Convert the found number to an integer\n",
    "    else:\n",
    "        return None  # Return None if no number is found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fef3e532-d471-4635-b743-947c402dbe35",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path = os.path.join(seqfish_dir, \"ALL_Roi\")  # Change this to the path where your fov folders are stored\n",
    "fov_data = []\n",
    "\n",
    "for fov_folder in os.listdir(base_path):\n",
    "    fov_folder_path = os.path.join(base_path, fov_folder)\n",
    "    if os.path.isdir(fov_folder_path):\n",
    "        # Loop through each ROI file in the fov folder\n",
    "        for roi_file in os.listdir(fov_folder_path):\n",
    "            if roi_file.endswith('.roi'):\n",
    "                key = roi_file.replace('.roi', '')\n",
    "                roi_file_path = os.path.join(fov_folder_path, roi_file)\n",
    "                polygon = process_roi_file(key, roi_file_path)\n",
    "                fov_data.append({\n",
    "                    'fov':  extract_fov_from_string(fov_folder),\n",
    "                    'cell': roi_file.replace('.roi', ''),\n",
    "                    'geometry': polygon\n",
    "                })\n",
    "\n",
    "cell_boundary_df = pd.DataFrame(fov_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f3b01b4-c042-4e70-9dd0-7ef88741b833",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### relabel cell name of polygons df to the standard name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15c59f4d-6fce-4702-861a-176516f518b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted = cell_boundary_df.sort_values(by=['fov', 'cell'])\n",
    "df_sorted['cell_id'] = df_sorted.groupby('fov').cumcount() + 1\n",
    "df_sorted['cell_id'] = df_sorted.apply(lambda x: f\"{x['fov']}_Cell_{x['cell_id']}\", axis=1)\n",
    "df_sorted.to_csv(\"/home/oneai/oneai-dda-spatialtr-visiumhd_analysis/cache/seqfish/cells_df.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c9e51e0-b001-4b31-a6cb-d9a9c8f32eb4",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Break cell polygons df to patches (based on fov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13e7bc10-1903-46ef-9042-9086b35259a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved patch_1.csv\n",
      "Saved patch_2.csv\n",
      "Saved patch_3.csv\n",
      "Saved patch_4.csv\n",
      "Saved patch_5.csv\n",
      "Saved patch_6.csv\n",
      "Saved patch_7.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1563651/2577681905.py:3: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  for fov, group in grouped:\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Create a df for each patch\n",
    "grouped = df_sorted.groupby(['fov'])\n",
    "for fov, group in grouped:\n",
    "    filename = f\"patch_{fov}.csv\"\n",
    "    output_loc = os.path.join(cells_df_chunks_dir, filename)\n",
    "    group.to_csv(output_loc)\n",
    "\n",
    "    print(f\"Saved {filename}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb4bebd9-bc07-44da-a02f-28d5ddc3c1ed",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Run ENACT bin-to-cell pipeline\n",
    "In the configs.yaml file: \n",
    "\n",
    "    Set \"analysis_name\" in the configs.yaml file to \"seqfish\".\n",
    "    Set \"run_synthetic\" to True.\n",
    "    Set \"bin_to_cell_method\" to one of these four: \"naive\", \"weighted_by_area\", \"weighted_by_gene\", or \"weighted_by_cluster\"\n",
    "\n",
    "Run `make run_enact`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a8fc042-2406-4db3-9617-7e3968ce8d28",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Evaluation of ENACT bin-to-cell results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01ff50d0-2993-42e9-98e9-fe478c32d605",
   "metadata": {},
   "source": [
    "To evaluate and compare the four bin-to-cell methods, please first complete the step above with all four methods. You can also only run the methods you are interested in and change the following code accordingly."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ef3fb7f-cc99-4f9e-b5cc-321412b08ddb",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Calculate precision, recall, and F1 for each bin2cell method"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d11287c-611d-49d2-a1e6-e12c14a973f5",
   "metadata": {},
   "source": [
    "Run this session with all the methods you have run with ENACT, change 'method' in the cell below to the one you want to evaluate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3c684f3-5b10-4bd2-8e1c-81d4cdb68ee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate all patches of ENACT results file \n",
    "method = \"weighted_by_gene\" # other methods: \"naive\", \"weighted_by_area\", \"weighted_by_cluster\" \n",
    "directory_path = os.path.join(enact_data_dir,method,\"bin_to_cell_assign\") \n",
    "output_file = os.path.join(enact_data_dir,method,\"bin_to_cell_assign/merged.csv\") \n",
    "\n",
    "concatenate_csv_files(directory_path, output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4580e62f-e2f3-4d1e-9a25-c483304a119e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "def concatenate_csv_files(directory_path, output_file):\n",
    "    dataframes = []\n",
    "\n",
    "    for filename in os.listdir(directory_path):\n",
    "        if filename.endswith('.csv'):\n",
    "            file_path = os.path.join(directory_path, filename)\n",
    "            df = pd.read_csv(file_path)\n",
    "            dataframes.append(df)\n",
    "    \n",
    "    concatenated_df = pd.concat(dataframes, ignore_index=True)\n",
    "    concatenated_df = concatenated_df.drop(columns = ['Unnamed: 0.1','Unnamed: 0'])\n",
    "    sorted_df = concatenated_df.sort_values(by='id')\n",
    "    sorted_df.to_csv(output_file, index=False)\n",
    "    print(f\"All CSV files have been concatenated into {output_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "263c024a-821e-4d15-9abd-d3463a8e34f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from shapely.geometry import Polygon\n",
    "\n",
    "def calculate_metrics(ground_truth_file, generated_file, eval_file):\n",
    "    # Load ground truth and generated data\n",
    "    ground_truth = pd.read_csv(ground_truth_file)\n",
    "    generated = pd.read_csv(generated_file)\n",
    "    generated.fillna(0)\n",
    "    # Ensure 'cell_id' is properly handled\n",
    "    if 'id' in generated.columns:\n",
    "        generated.rename(columns={'id': 'new_cell_name'}, inplace=True)\n",
    "\n",
    "    # Merge data on 'cell_id'\n",
    "    merged = pd.merge(\n",
    "        ground_truth, generated, on='new_cell_name', how='outer', suffixes=('_gt', '_gen')\n",
    "    ).fillna(0)\n",
    "    # print(merged)\n",
    "\n",
    "    # Identify common gene features\n",
    "    gt_columns = merged.filter(like='_gt').columns\n",
    "    gen_columns = merged.filter(like='_gen').columns\n",
    "\n",
    "    common_genes = set(gt_columns).intersection(gen_columns)\n",
    "\n",
    "    # Reorder columns based on common genes\n",
    "    ordered_gt_columns = sorted(gt_columns)\n",
    "    ordered_gen_columns = sorted(gen_columns)\n",
    "    \n",
    "\n",
    "    # Extract aligned matrices for ground truth and generated data\n",
    "    ground_truth_aligned = merged[['new_cell_name'] + [col for col in ordered_gt_columns if col in gt_columns]].values\n",
    "    generated_aligned = merged[['new_cell_name'] + [col for col in ordered_gen_columns if col in gen_columns]].values\n",
    "    \n",
    "    print(ground_truth_aligned)\n",
    "    print(generated_aligned)\n",
    "    # Ensure matrices are aligned and have the same shape\n",
    "    if ground_truth_aligned.shape[1] != generated_aligned.shape[1]:\n",
    "        raise ValueError(\"The aligned matrices must have the same shape!\")\n",
    "\n",
    "    ground_truth_aligned = ground_truth_aligned[:, 1:]  # Exclude cell_ids\n",
    "    generated_aligned = generated_aligned[:, 1:]     \n",
    "\n",
    "    num_cells = (ground_truth.iloc[:, 1:] != 0).any(axis=1).sum()\n",
    "    tp = np.sum(np.minimum(generated_aligned, ground_truth_aligned), axis=1)\n",
    "    predicted = np.sum(generated_aligned, axis=1)\n",
    "    actual = np.sum(ground_truth_aligned, axis=1)\n",
    "\n",
    "    # Calculate precision, recall, and F1 score for each row\n",
    "    precision = tp / predicted\n",
    "    recall = tp / actual\n",
    "    f1_score = 2 * (precision * recall) / (precision + recall)\n",
    "    \n",
    "\n",
    "   # Add a column called 'Method' where all rows have the same entry\n",
    "    method_column = np.full((precision.shape[0],), 'Naive')  # Replace 'YourMethodName' with the actual method name\n",
    "\n",
    "    df = pd.DataFrame({\n",
    "        'Precision': precision,\n",
    "        'Recall': recall,\n",
    "        'F1 Score': f1_score,\n",
    "        'Method': method_column\n",
    "    })\n",
    "\n",
    "\n",
    "    df.to_csv(eval_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db707de6-8da9-495e-82a0-69c009cf1475",
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth_file = os.path.join(seqfish_dir, \"groundtruth.csv\")\n",
    "generated_file = os.path.join(enact_data_dir,method,\"bin_to_cell_assign/merged.csv\")\n",
    "eval_file = os.path.join(enact_data_dir,method,\"eval.csv\") \n",
    "\n",
    "calculate_metrics(ground_truth_file, generated_file, eval_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cd9470e-8410-4510-9165-cebd466ab343",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Create violin plots comparing four bin2cell methods"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b78b6e7d-0e57-46fd-bddb-c701750a625b",
   "metadata": {},
   "source": [
    "The following cells would create violin plots for all four methods in order to better compare the results. You can choose to only compare the ones you have run by changing the 'file_names' list to only include those."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dc3b4e5-798b-4d0f-a243-bc619daa6f50",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [os.path.join(enact_data_dir,\"naive/eval.csv\"), \n",
    "              os.path.join(enact_data_dir,\"weighted_by_area/eval.csv\"), \n",
    "              os.path.join(enact_data_dir,\"weighted_by_gene/eval.csv\"),\n",
    "              os.path.join(enact_data_dir,\"weighted_by_cluster/eval.csv\")]  # Replace with actual file paths\n",
    "\n",
    "# Read and concatenate all files\n",
    "df_list = [pd.read_csv(file) for file in file_names]\n",
    "metrics_df = pd.concat(df_list, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a9f977f-6530-446a-9aa4-57d0cbbca85b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize the distributions using violin plots\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "# Create a figure with subplots for each metric\n",
    "fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n",
    "\n",
    "# Precision Violin Plot\n",
    "sns.violinplot(x='Method', y='Precision', data=metrics_df, ax=axes[0], inner='quartile', palette='Set2')\n",
    "axes[0].set_title('Precision')\n",
    "axes[0].set_xlabel('Method')\n",
    "axes[0].set_ylabel('value')\n",
    "axes[0].set_ylim(0.8,1)\n",
    "axes[0].tick_params(axis='x', labelsize=8)  # Adjust the font size here\n",
    "\n",
    "# Recall Violin Plot\n",
    "sns.violinplot(x='Method', y='Recall', data=metrics_df, ax=axes[1], inner='quartile', palette='Set2')\n",
    "axes[1].set_title('Recall')\n",
    "axes[1].set_xlabel('Method')\n",
    "axes[1].set_ylabel('value')\n",
    "axes[1].set_ylim(0.8,1)\n",
    "axes[1].tick_params(axis='x', labelsize=8)  # Adjust the font size here\n",
    "\n",
    "# F1 Score Violin Plot\n",
    "sns.violinplot(x='Method', y='F1 Score', data=metrics_df, ax=axes[2], inner='quartile', palette='Set2')\n",
    "axes[2].set_title('F1 Score')\n",
    "axes[2].set_xlabel('Method')\n",
    "axes[2].set_ylabel('value')\n",
    "axes[2].set_ylim(0.8,1)\n",
    "axes[2].tick_params(axis='x', labelsize=8)  # Adjust the font size here\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}