docs: Added RST files for notebooks (#101)

docs/source/conf.py

@@ -61,7 +61,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
+exclude_patterns = ['notebooks/*.rst']
 
 
 # The name of the Pygments (syntax highlighting) style to use.

docs/source/notebooks/latency_benchmark.rst

@@ -0,0 +1,149 @@
+Installation
+============
+
+In this tutorial, you will need the entire project codebase. So first,
+we clone the project’s GitHub repository and install from source.
+
+.. code-block::python
+
+ >>> !git clone https://github.com/frgfm/torch-cam.git
+ >>> !pip install -e torch-cam/.
+
+
+Hardware information
+====================
+
+GPU information
+---------------
+
+To be able to run the benchmark on GPU, you need to have the correct
+driver and CUDA installation. If you get a message starting with: >
+NVIDIA-SMI has failed…
+
+The script will be running on CPU as PyTorch isn’t able to access any
+CUDA-capable device.
+
+.. code-block::python
+
+ >>> !nvidia-smi
+ NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver. Make sure that the latest NVIDIA driver is installed and running.
+
+
+CPU information
+---------------
+
+Latency will vary greatly depending on the capabilities of your CPU.
+Some models are optimized for CPU architectures (MobileNet V3 for
+instance), while others were only designed for GPU and will thus yield
+poor latency when run on CPU.
+
+.. code-block::python
+
+ >>> !lscpu
+ Architecture: x86_64
+ CPU op-mode(s): 32-bit, 64-bit
+ Byte Order: Little Endian
+ CPU(s): 2
+ On-line CPU(s) list: 0,1
+ Thread(s) per core: 2
+ Core(s) per socket: 1
+ Socket(s): 1
+ NUMA node(s): 1
+ Vendor ID: AuthenticAMD
+ CPU family: 23
+ Model: 49
+ Model name: AMD EPYC 7B12
+ Stepping: 0
+ CPU MHz: 2249.998
+ BogoMIPS: 4499.99
+ Hypervisor vendor: KVM
+ Virtualization type: full
+ L1d cache: 32K
+ L1i cache: 32K
+ L2 cache: 512K
+ L3 cache: 16384K
+ NUMA node0 CPU(s): 0,1
+ Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext ssbd ibrs ibpb stibp vmmcall fsgsbase tsc_adjust bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr arat npt nrip_save umip rdpid
+
+
+Usage
+=====
+
+The latency evaluation script provides several options for you to play
+with: change the input size, the architecture or the CAM method to
+better reflect your use case.
+
+.. code-block::python
+
+ >>> !cd torch-cam/ && python scripts/eval_latency.py --help
+ usage: eval_latency.py [-h] [--arch ARCH] [--size SIZE]
+ [--class-idx CLASS_IDX] [--device DEVICE] [--it IT]
+ method
+ 
+ CAM method latency benchmark
+ 
+ positional arguments:
+ method CAM method to use
+ 
+ optional arguments:
+ -h, --help show this help message and exit
+ --arch ARCH Name of the torchvision architecture (default:
+ resnet18)
+ --size SIZE The image input size (default: 224)
+ --class-idx CLASS_IDX
+ Index of the class to inspect (default: 232)
+ --device DEVICE Default device to perform computation on (default:
+ None)
+ --it IT Number of iterations to run (default: 100)
+
+
+Architecture designed for GPU
+-----------------------------
+
+Let’s benchmark the latency of CAM methods with the popular ResNet
+architecture
+
+.. code-block::python
+
+ >>> !cd torch-cam/ && python scripts/eval_latency.py SmoothGradCAMpp --arch resnet18
+ Downloading: "https://download.pytorch.org/models/resnet18-f37072fd.pth" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth
+ 100% 44.7M/44.7M [00:00<00:00, 85.9MB/s]
+ /usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /pytorch/c10/core/TensorImpl.h:1156.)
+ return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
+ WARNING:root:no value was provided for `target_layer`, thus set to 'layer4'.
+ SmoothGradCAMpp w/ resnet18 (100 runs on (224, 224) inputs)
+ mean 1143.17ms, std 36.79ms
+
+
+.. code-block::python
+
+ >>> !cd torch-cam/ && python scripts/eval_latency.py LayerCAM --arch resnet18
+ /usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /pytorch/c10/core/TensorImpl.h:1156.)
+ return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
+ WARNING:root:no value was provided for `target_layer`, thus set to 'layer4'.
+ LayerCAM w/ resnet18 (100 runs on (224, 224) inputs)
+ mean 189.64ms, std 8.82ms
+
+
+Architecture designed for CPU
+-----------------------------
+
+As mentioned, we’ll consider MobileNet V3 here.
+
+.. code-block::python
+
+ >>> !cd torch-cam/ && python scripts/eval_latency.py SmoothGradCAMpp --arch mobilenet_v3_large
+ Downloading: "https://download.pytorch.org/models/mobilenet_v3_large-8738ca79.pth" to /root/.cache/torch/hub/checkpoints/mobilenet_v3_large-8738ca79.pth
+ 100% 21.1M/21.1M [00:00<00:00, 71.5MB/s]
+ WARNING:root:no value was provided for `target_layer`, thus set to 'features.4.block.1'.
+ SmoothGradCAMpp w/ mobilenet_v3_large (100 runs on (224, 224) inputs)
+ mean 762.18ms, std 26.95ms
+
+
+.. code-block::python
+
+ >>> !cd torch-cam/ && python scripts/eval_latency.py LayerCAM --arch mobilenet_v3_large
+ WARNING:root:no value was provided for `target_layer`, thus set to 'features.4.block.1'.
+ LayerCAM w/ mobilenet_v3_large (100 runs on (224, 224) inputs)
+ mean 148.76ms, std 7.86ms
+

docs/source/notebooks/quicktour.rst

@@ -0,0 +1,177 @@
+Installation
+============
+
+Install all the dependencies to make the most out of TorchCAM
+
+.. code-block::python
+
+ >>> !pip install torchvision matplotlib
+
+
+Latest stable release
+---------------------
+
+.. code-block:: python
+
+ >>> !pip install torch-cam
+
+From source
+-----------
+
+.. code-block:: python
+
+ >>> # Install the most up-to-date version from GitHub
+ >>> !pip install -e git+https://github.com/frgfm/torch-cam.git#egg=torchcam
+
+
+Now go to ``Runtime/Restart runtime`` for your changes to take effect!
+
+Basic usage
+===========
+
+.. code-block:: python
+
+ >>> # Download an image
+ >>> !wget https://www.woopets.fr/assets/races/000/066/big-portrait/border-collie.jpg
+ >>> # Set this to your image path if you wish to run it on your own data
+ >>> img_path = "border-collie.jpg"
+
+
+.. code-block:: python
+
+ >>> # Instantiate your model here
+ >>> from torchvision.models import resnet18
+ >>> model = resnet18(pretrained=True).eval()
+
+
+
+Illustrate your classifier capabilities
+---------------------------------------
+
+.. code-block:: python
+
+ >>> %matplotlib inline
+ >>> # All imports
+ >>> from torchvision.io.image import read_image
+ >>> from torchvision.transforms.functional import normalize, resize, to_pil_image
+ >>> import matplotlib.pyplot as plt
+ >>> from torchcam.cams import SmoothGradCAMpp, LayerCAM
+ >>> from torchcam.utils import overlay_mask
+
+.. code-block:: python
+
+ >>> cam_extractor = SmoothGradCAMpp(model)
+ >>> # Get your input
+ >>> img = read_image(img_path)
+ >>> # Preprocess it for your chosen model
+ >>> input_tensor = normalize(resize(img, (224, 224)) / 255., [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+ >>> # Preprocess your data and feed it to the model
+ >>> out = model(input_tensor.unsqueeze(0))
+ >>> # Retrieve the CAM by passing the class index and the model output
+ >>> cams = cam_extractor(out.squeeze(0).argmax().item(), out)
+
+.. code-block:: python
+
+ >>> # Notice that there is one CAM per target layer (here only 1)
+ >>> for cam in cams:
+ >>> print(cam.shape)
+ torch.Size([7, 7])
+
+
+.. code-block:: python
+
+ >>> # The raw CAM
+ >>> for name, cam in zip(cam_extractor.target_names, cams):
+ >>> plt.imshow(cam.numpy()); plt.axis('off'); plt.title(name); plt.show()
+
+
+.. code-block:: python
+
+ >>> # Overlayed on the image
+ >>> for name, cam in zip(cam_extractor.target_names, cams):
+ >>> result = overlay_mask(to_pil_image(img), to_pil_image(cam, mode='F'), alpha=0.5)
+ >>> plt.imshow(result); plt.axis('off'); plt.title(name); plt.show()
+
+
+.. code-block:: python
+
+ >>> # Once you're finished, clear the hooks on your model
+ >>> cam_extractor.clear_hooks()
+
+Advanced tricks
+===============
+
+Extract localization cues
+-------------------------
+
+.. code-block::python
+
+ >>> import torch
+ >>> from torch.nn.functional import softmax, interpolate
+
+.. code-block::python
+
+ >>> # Retrieve the CAM from several layers at the same time
+ >>> cam_extractor = LayerCAM(model)
+ >>> # Preprocess your data and feed it to the model
+ >>> out = model(input_tensor.unsqueeze(0))
+ >>> print(softmax(out, dim=1).max())
+ tensor(0.9115, grad_fn=<MaxBackward1>)
+
+
+.. code-block::python
+
+ >>> cams = cam_extractor(out.squeeze(0).argmax().item(), out)
+
+.. code-block::python
+
+ >>> # Resize it
+ >>> resized_cams = [resize(to_pil_image(cam), img.shape[-2:]) for cam in cams]
+ >>> segmaps = [to_pil_image((resize(cam.unsqueeze(0), img.shape[-2:]).squeeze(0) >= 0.5).to(dtype=torch.float32)) for cam in cams]
+ >>> # Plot it
+ >>> for name, cam, seg in zip(cam_extractor.target_names, resized_cams, segmaps):
+ >>> _, axes = plt.subplots(1, 2)
+ >>> axes[0].imshow(cam); axes[0].axis('off'); axes[0].set_title(name)
+ >>> axes[1].imshow(seg); axes[1].axis('off'); axes[1].set_title(name)
+ >>> plt.show()
+
+
+Fuse CAMs from multiple layers
+------------------------------
+
+.. code-block::python
+
+ >>> # Retrieve the CAM from several layers at the same time
+ >>> cam_extractor = LayerCAM(model, ["layer2", "layer3", "layer4"])
+ >>> # Preprocess your data and feed it to the model
+ >>> out = model(input_tensor.unsqueeze(0))
+ >>> # Retrieve the CAM by passing the class index and the model output
+ >>> cams = cam_extractor(out.squeeze(0).argmax().item(), out)
+
+.. code-block::python
+
+ >>> # This time, there are several CAMs
+ >>> for cam in cams:
+ >>> print(cam.shape)
+ torch.Size([14, 14])
+ torch.Size([7, 7])
+
+
+.. code-block::python
+
+ >>> # The raw CAM
+ >>> _, axes = plt.subplots(1, len(cam_extractor.target_names))
+ >>> for idx, name, cam in zip(range(len(cam_extractor.target_names)), cam_extractor.target_names, cams):
+ >>> axes[idx].imshow(cam.numpy()); axes[idx].axis('off'); axes[idx].set_title(name);
+ >>> plt.show()
+
+
+.. code-block::python
+
+ >>> # Let's fuse them
+ >>> fused_cam = cam_extractor.fuse_cams(cams)
+ >>> # Plot the raw version
+ >>> plt.imshow(fused_cam.numpy()); plt.axis('off'); plt.title(" + ".join(cam_extractor.target_names)); plt.show()
+ >>> # Plot the overlayed version
+ >>> result = overlay_mask(to_pil_image(img), to_pil_image(fused_cam, mode='F'), alpha=0.5)
+ >>> plt.imshow(result); plt.axis('off'); plt.title(" + ".join(cam_extractor.target_names)); plt.show()