Panoptic segmentation

app/packages/core/src/components/Modal/TooltipInfo.tsx

@@ -645,6 +645,58 @@ const KeypointInfo = ({ detail }) => {
   );
 };
 
+const PanopticSegmentationInfo = ({ detail }) => {
+  console.log(detail);
+  const targetValue = useTarget(detail.field, detail.target);
+  const hideTargetValue = detail.color === undefined;
+
+  return (
+    <AttrBlock style={{ borderColor: detail.color }}>
+      {!hideTargetValue &&
+        (targetValue ? (
+          <ContentItem
+            key={"target-value"}
+            field={detail.field}
+            name={"label"}
+            value={targetValue}
+          />
+        ) : (
+          <ContentItem
+            key={"pixel-value"}
+            field={detail.field}
+            name={"pixel"}
+            value={detail.target}
+          />
+        ))}
+      {detail.instance > 0 && (
+        <ContentItem
+          key={"instance-value"}
+          field={detail.field}
+          name={"instance"}
+          value={detail.instance}
+        />
+      )}
+      <AttrInfo
+        field={detail.field}
+        label={detail.label}
+        labelType={detail.type}
+      />
+    </AttrBlock>
+  );
+};
+
+const PolylineInfo = ({ detail }) => {
+  return (
+    <AttrBlock style={{ borderColor: detail.color }}>
+      <AttrInfo
+        field={detail.field}
+        label={detail.label}
+        labelType={detail.type}
+      />
+    </AttrBlock>
+  );
+};
+
 const RegressionInfo = ({ detail }) => {
   return (
     <AttrBlock style={{ borderColor: detail.color }}>
@@ -679,18 +731,14 @@ const SegmentationInfo = ({ detail }) => {
             value={detail.target}
           />
         ))}
-      <AttrInfo
-        field={detail.field}
-        label={detail.label}
-        labelType={detail.type}
-      />
-    </AttrBlock>
-  );
-};
-
-const PolylineInfo = ({ detail }) => {
-  return (
-    <AttrBlock style={{ borderColor: detail.color }}>
+      {detail.instance > 0 && (
+        <ContentItem
+          key={"instance-value"}
+          field={detail.field}
+          name={"instance"}
+          value={detail.instance}
+        />
+      )}
       <AttrInfo
         field={detail.field}
         label={detail.label}
@@ -705,6 +753,7 @@ const OVERLAY_INFO = {
   Detection: DetectionInfo,
   Heatmap: HeatmapInfo,
   Keypoint: KeypointInfo,
+  PanopticSegmentation: PanopticSegmentationInfo,
   Polyline: PolylineInfo,
   Regression: RegressionInfo,
   Segmentation: SegmentationInfo,
@@ -715,6 +764,7 @@ const HIDDEN_LABELS = {
   Detection: ["bounding_box", "mask"],
   Heatmap: ["map"],
   Keypoint: ["points", "occluded", "confidence"],
+  PanopticSegmentation: ["mask"],
   Polyline: ["points"],
   Regression: [],
   Segmentation: ["mask"],

app/packages/looker/package.json

@@ -21,6 +21,8 @@
         "@ungap/event-target": "^0.2.2",
         "@xmldom/xmldom": "^0.8.6",
         "copy-to-clipboard": "^3.3.1",
+        "decode-tiff": "^0.2.1",
+        "geotiff": "^2.1.3",
         "immutable": "^4.0.0-rc.12",
         "lodash": "^4.17.21",
         "lru-cache": "^11.0.1",
@@ -32,6 +34,7 @@
         "@rollup/plugin-inject": "^5.0.2",
         "@types/color-string": "^1.5.0",
         "@types/lru-cache": "^7.10.10",
+        "@types/utif": "^3.0.5",
         "@types/uuid": "^8.3.0",
         "buffer": "^6.0.3",
         "prettier": "^2.7.1",

app/packages/looker/src/overlays/base.ts

@@ -3,8 +3,8 @@
  */
 
 import { getCls, sizeBytesEstimate } from "@fiftyone/utilities";
-import { OverlayMask } from "../numpy";
 import type { BaseState, Coordinates, NONFINITE } from "../state";
+import type { OverlayMask } from "../worker/decoders/types";
 import { getLabelColor, shouldShowLabelTag } from "./util";
 
 // in numerical order (CONTAINS_BORDER takes precedence over CONTAINS_CONTENT)
@@ -30,6 +30,7 @@ export interface PointInfo<Label extends BaseLabel = BaseLabel> {
     attributes: [string, unknown][];
     coordinates: Coordinates;
   };
+  instance?: bigint | number;
   target?: number;
   type: string;
 }

app/packages/looker/src/overlays/detection.ts

@@ -2,21 +2,16 @@
  * Copyright 2017-2025, Voxel51, Inc.
  */
 import { NONFINITES } from "@fiftyone/utilities";
-
 import { INFO_COLOR } from "../constants";
-import { BaseState, BoundingBox, Coordinates, NONFINITE } from "../state";
+import type { BaseState, BoundingBox, Coordinates, NONFINITE } from "../state";
 import { distanceFromLineSegment } from "../util";
-import {
-  CONTAINS,
-  CoordinateOverlay,
-  LabelMask,
-  PointInfo,
-  RegularLabel,
-} from "./base";
+import type { LabelMask, PointInfo, RegularLabel } from "./base";
+import { CONTAINS, CoordinateOverlay } from "./base";
 import { t } from "./util";
 
 export interface DetectionLabel extends RegularLabel {
   mask?: LabelMask;
+  mask_path?: string;
   bounding_box: BoundingBox;
 
   // valid for 3D bounding boxes
@@ -166,17 +161,17 @@ export default class DetectionOverlay<
     let text =
       this.label.label && state.options.showLabel ? `${this.label.label}` : "";
 
-    if (state.options.showIndex && !isNaN(this.label.index)) {
-      text.length && (text += " ");
+    if (state.options.showIndex && !Number.isNaN(this.label.index)) {
+      if (text.length) text += " ";
       text += `${Number(this.label.index).toLocaleString()}`;
     }
 
     if (
       state.options.showConfidence &&
-      (!isNaN(this.label.confidence as number) ||
+      (!Number.isNaN(this.label.confidence as number) ||
         NONFINITES.has(this.label.confidence as NONFINITE))
     ) {
-      text.length && (text += " ");
+      if (text.length) text += " ";
       text += `(${
         typeof this.label.confidence === "number"
           ? Number(this.label.confidence).toFixed(2)
@@ -268,7 +263,7 @@ export default class DetectionOverlay<
 
 export const getDetectionPoints = (labels: DetectionLabel[]): Coordinates[] => {
   let points: Coordinates[] = [];
-  labels.forEach((label) => {
+  for (const label of labels) {
     const [tlx, tly, w, h] = label.bounding_box;
     points = [
       ...points,
@@ -277,6 +272,6 @@ export const getDetectionPoints = (labels: DetectionLabel[]): Coordinates[] => {
       [tlx + w, tly + h],
       [tlx, tly + h],
     ];
-  });
+  }
   return points;
 };

app/packages/looker/src/overlays/heatmap.ts

@@ -8,22 +8,22 @@ import {
   getRGBAColor,
   sizeBytesEstimate,
 } from "@fiftyone/utilities";
-import { ARRAY_TYPES, TypedArray } from "../numpy";
-import { BaseState, Coordinates } from "../state";
+import type { BaseState, Coordinates } from "../state";
 import { isFloatArray } from "../util";
-import { clampedIndex } from "../worker/painter";
-import {
+import type { TypedArray } from "../worker/decoders/types";
+import { ARRAY_TYPES } from "../worker/decoders/types";
+import { clampedIndex } from "../worker/painters/utils";
+import type {
   BaseLabel,
-  CONTAINS,
   LabelMask,
   Overlay,
   PointInfo,
   SelectData,
-  isShown,
 } from "./base";
+import { CONTAINS, isShown } from "./base";
 import { strokeCanvasRect, t } from "./util";
 
-interface HeatmapLabel extends BaseLabel {
+export interface HeatmapLabel extends BaseLabel {
   map?: LabelMask;
   range?: [number, number];
 }
@@ -103,7 +103,7 @@ export default class HeatmapOverlay<State extends BaseState>
     if (this.containsPoint(state)) {
       return 0;
     }
-    return Infinity;
+    return Number.Infinity;
   }
 
   getPointInfo(state: Readonly<State>): PointInfo<HeatmapInfo> {
@@ -212,7 +212,7 @@ export default class HeatmapOverlay<State extends BaseState>
   }
 }
 
-export const getHeatmapPoints = (labels: HeatmapLabel[]): Coordinates[] => {
+export const getHeatmapPoints = (_: HeatmapLabel[]): Coordinates[] => {
   return [
     [0, 0],
     [0, 1],

app/packages/looker/src/overlays/index.ts

@@ -1,27 +1,32 @@
 /**
  * Copyright 2017-2025, Voxel51, Inc.
  */
+import type { Schema } from "@fiftyone/utilities";
 import {
   DYNAMIC_EMBEDDED_DOCUMENT_FIELD,
   EMBEDDED_DOCUMENT_FIELD,
+  LABEL_LISTS_MAP,
   getCls,
   getFieldInfo,
-  LABEL_LISTS_MAP,
-  Schema,
 } from "@fiftyone/utilities";
 import { LABEL_TAGS_CLASSES } from "../constants";
-import { BaseState } from "../state";
-import { Overlay } from "./base";
-import {
+import type { BaseState } from "../state";
+import type { Overlay } from "./base";
+import type {
   ClassificationLabel,
-  ClassificationsOverlay,
   Labels,
   TemporalDetectionLabel,
+} from "./classifications";
+import {
+  ClassificationsOverlay,
   TemporalDetectionOverlay,
 } from "./classifications";
 import DetectionOverlay, { getDetectionPoints } from "./detection";
 import HeatmapOverlay, { getHeatmapPoints } from "./heatmap";
 import KeypointOverlay, { getKeypointPoints } from "./keypoint";
+import PanopticSegmentationOverlay, {
+  getPanopticSegmentationPoints,
+} from "./panoptic";
 import PolylineOverlay, { getPolylinePoints } from "./polyline";
 import SegmentationOverlay, { getSegmentationPoints } from "./segmentation";
 
@@ -41,6 +46,7 @@ export const FROM_FO = {
   Heatmap: fromLabel(HeatmapOverlay),
   Keypoint: fromLabel(KeypointOverlay),
   Keypoints: fromLabelList(KeypointOverlay, "keypoints"),
+  PanopticSegmentation: fromLabel(PanopticSegmentationOverlay),
   Polyline: fromLabel(PolylineOverlay),
   Polylines: fromLabelList(PolylineOverlay, "polylines"),
   Segmentation: fromLabel(SegmentationOverlay),
@@ -52,9 +58,10 @@ export const POINTS_FROM_FO = {
   Heatmap: (label) => getHeatmapPoints(label ? [label] : []),
   Keypoint: (label) => getKeypointPoints(label ? [label] : []),
   Keypoints: (label) => getKeypointPoints(label?.keypoints ?? []),
+  PanopticSegmentation: (label) => getSegmentationPoints(label ? [label] : []),
   Polyline: (label) => getPolylinePoints(label ? [label] : []),
   Poylines: (label) => getPolylinePoints(label?.polylines ?? []),
-  Segmentation: (label) => getSegmentationPoints(label ? [label] : []),
+  Segmentation: (label) => getPanopticSegmentationPoints(label ? [label] : []),
 };
 
 const LABEL_LISTS = LABEL_LISTS_MAP;