organizing

app/packages/looker/package.json

@@ -18,21 +18,27 @@
         "README.md"
     ],
     "dependencies": {
+        "@jimp/tiff": "^0.22.12",
         "@ungap/event-target": "^0.2.2",
         "@xmldom/xmldom": "^0.8.6",
         "copy-to-clipboard": "^3.3.1",
         "decode-tiff": "^0.2.1",
+        "geotiff": "^2.1.3",
         "immutable": "^4.0.0-rc.12",
         "lodash": "^4.17.21",
         "lru-cache": "^11.0.1",
         "mime": "^2.5.2",
         "monotone-convex-hull-2d": "^1.0.1",
+        "tiff": "^6.1.1",
+        "utif": "^3.1.0",
+        "utif2": "^4.1.0",
         "uuid": "^8.3.2"
     },
     "devDependencies": {
         "@rollup/plugin-inject": "^5.0.2",
         "@types/color-string": "^1.5.0",
         "@types/lru-cache": "^7.10.10",
+        "@types/utif": "^3.0.5",
         "@types/uuid": "^8.3.0",
         "buffer": "^6.0.3",
         "prettier": "^2.7.1",

app/packages/looker/src/state.ts

@@ -2,11 +2,15 @@
  * Copyright 2017-2024, Voxel51, Inc.
  */
 
-import { BufferManager } from "@fiftyone/utilities";
-import { ImaVidFramesController } from "./lookers/imavid/controller";
-import { Overlay } from "./overlays/base";
-
-import { AppError, COLOR_BY, Schema, Stage } from "@fiftyone/utilities";
+import type {
+  AppError,
+  BufferManager,
+  COLOR_BY,
+  Schema,
+  Stage,
+} from "@fiftyone/utilities";
+import type { ImaVidFramesController } from "./lookers/imavid/controller";
+import type { Overlay } from "./overlays/base";
 
 export type Optional<T> = {
   [P in keyof T]?: Optional<T[P]>;
@@ -70,27 +74,6 @@ export type OrthogrpahicProjectionMetadata = {
   normal: [number, number, number];
 };
 
-export type GenericLabel = {
-  [labelKey: string]: {
-    [field: string]: unknown;
-  };
-  // todo: add other label types
-};
-
-export type Sample = {
-  metadata: {
-    width: number;
-    height: number;
-    mime_type?: string;
-  };
-  _id: string;
-  id: string;
-  filepath: string;
-  tags: string[];
-  _label_tags: string[];
-  _media_type: "image" | "video" | "point-cloud" | "3d";
-} & GenericLabel;
-
 export interface LabelData {
   labelId: string;
   field: string;
@@ -127,8 +110,8 @@ export type Action<State extends BaseState> = (
 ) => void;
 
 export enum ControlEventKeyType {
-  HOLD,
-  KEY_DOWN,
+  HOLD = "HOLD",
+  KEY_DOWN = "KEY_DOWN",
 }
 export interface Control<State extends BaseState = BaseState> {
   eventKeys?: string | string[];

app/packages/looker/src/worker/decoders/canvas.ts

@@ -1,5 +1,4 @@
-import type { OverlayMask } from "../../numpy";
-import { enqueueFetch } from "../pooled-fetch";
+import type { OverlayMask } from "./types";
 
 const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
 /**
@@ -34,13 +33,7 @@ const getPngcolorType = async (blob: Blob): Promise<number | undefined> => {
   return colorType;
 };
 
-const decodeWithCanvas = async (url: string): Promise<OverlayMask> => {
-  const overlayImageFetchResponse = await enqueueFetch({
-    url,
-    options: { priority: "low" },
-  });
-  const blob = await overlayImageFetchResponse.blob();
-
+const decodeWithCanvas = async (blob: Blob): Promise<OverlayMask> => {
   let channels = 4;
 
   if (blob.type === "image/png") {

app/packages/looker/src/worker/decoders/index.ts

@@ -1,51 +1,35 @@
 import { getSampleSrc } from "@fiftyone/state/src/recoil/utils";
 import { DETECTION, DETECTIONS } from "@fiftyone/utilities";
 import type { Coloring, CustomizeColor } from "../..";
-import type { OverlayMask } from "../../numpy";
 import type { Colorscale } from "../../state";
+import { enqueueFetch } from "../pooled-fetch";
 import { getOverlayFieldFromCls } from "../shared";
 import decodeCanvas from "./canvas";
+import decodeInline from "./inline";
 import decodeTiff from "./tiff";
-
-type Decoder = (url: string) => Promise<OverlayMask>;
-
-const IMAGE_DECODERS: { [key: string]: Decoder } = {
-  jpeg: decodeCanvas,
-  jpg: decodeCanvas,
-  png: decodeCanvas,
-  tif: decodeTiff,
-  tiff: decodeTiff,
-};
-
-async function decodeImage(url: string) {
-  const extension = url.split(".").slice(-1)[0];
-  return IMAGE_DECODERS[extension](url);
-}
-
-export type IntermediateMask = {
-  data: OverlayMask;
-  image: ArrayBuffer;
-};
-
-interface Decodeable {
-  mask_path?: string;
-  map_path?: string;
-  detections?: { mask_path?: string }[];
-}
+import type { BlobDecoder, Decodeables } from "./types";
 
 interface DecodeParameters {
   cls: string;
   coloring: Coloring;
   colorscale: Colorscale;
   customizeColorSetting: CustomizeColor[];
   field: string;
-  label: Decodeable;
+  label: Decodeables;
   maskPathDecodingPromises?: Promise<void>[];
   maskTargetsBuffers?: ArrayBuffer[];
   overlayCollectionProcessingParams?: { idx: number; cls: string };
   sources: { [path: string]: string };
 }
 
+const IMAGE_DECODERS: { [key: string]: BlobDecoder } = {
+  jpeg: decodeCanvas,
+  jpg: decodeCanvas,
+  png: decodeCanvas,
+  tif: decodeTiff,
+  tiff: decodeTiff,
+};
+
 /**
  * Some label types (example: segmentation, heatmap) can have their overlay
  * data stored on-disk, we want to impute the relevant mask property of these
@@ -74,21 +58,31 @@ const decode = async ({ field, label, ...params }: DecodeParameters) => {
   const overlayPathField = overlayFields.disk;
   const overlayField = overlayFields.canonical;
 
+  const data = label[overlayField];
+  if (typeof data === "string") {
+    const intermediate = decodeInline(data);
+    if (!intermediate) {
+      return;
+    }
+
+    params.maskTargetsBuffers.push(intermediate.data.buffer);
+    label[overlayField] = intermediate;
+    return;
+  }
+
   if (!Object.hasOwn(label, overlayPathField)) {
     return;
   }
 
-  // it's possible we're just re-coloring, in which case re-init mask image
-  // and set bitmap to null
-  if (label[overlayField]?.bitmap && !label[overlayField]?.image) {
+  if (data?.bitmap && !data?.image) {
     const height = label[overlayField].bitmap.height;
     const width = label[overlayField].bitmap.width;
 
     // close the copied bitmap
-    label[overlayField].bitmap.close();
-    label[overlayField].bitmap = null;
+    data.bitmap.close();
+    data.bitmap = null;
 
-    label[overlayField].image = new ArrayBuffer(height * width * 4);
+    data.image = new ArrayBuffer(height * width * 4);
     return;
   }
 
@@ -110,7 +104,13 @@ const decode = async ({ field, label, ...params }: DecodeParameters) => {
 
   // convert absolute file path to a URL that we can "fetch" from
   const overlayImageUrl = getSampleSrc(source || label[overlayPathField]);
-  const overlayMask = await decodeImage(overlayImageUrl);
+  const overlayImageFetchResponse = await enqueueFetch({
+    url: overlayImageUrl,
+    options: { priority: "low" },
+  });
+  const blob = await overlayImageFetchResponse.blob();
+  const extension = overlayImageUrl.split(".").slice(-1)[0];
+  const overlayMask = await IMAGE_DECODERS[extension](blob);
   const [overlayHeight, overlayWidth] = overlayMask.shape;
 
   // set the `mask` property for this label
@@ -120,7 +120,7 @@ const decode = async ({ field, label, ...params }: DecodeParameters) => {
   label[overlayField] = {
     data: overlayMask,
     image: new ArrayBuffer(overlayWidth * overlayHeight * 4),
-  } as IntermediateMask;
+  };
 
   // no need to transfer image's buffer
   // since we'll be constructing ImageBitmap and transfering that

app/packages/looker/src/worker/decoders/inline.ts

@@ -0,0 +1,23 @@
+import { deserialize } from "./numpy";
+import type { InlineDecodeable, IntermediateMask } from "./types";
+
+export default (inline: InlineDecodeable): IntermediateMask | null => {
+  let base64: string;
+  if (typeof inline === "string") {
+    base64 = inline;
+  } else if (typeof inline?.$binary?.base64 === "string") {
+    base64 = inline.$binary.base64;
+  }
+
+  if (!base64) {
+    return null;
+  }
+
+  const data = deserialize(base64);
+  const [height, width] = data.shape;
+
+  return {
+    data,
+    image: new ArrayBuffer(width * height * 4),
+  };
+};

app/packages/looker/src/numpy.ts → app/packages/looker/src/worker/decoders/numpy.ts

@@ -4,42 +4,7 @@
 
 import { Buffer } from "buffer";
 import pako from "./pako.js";
-
-export { deserialize };
-
-export const ARRAY_TYPES = {
-  Uint8Array,
-  Uint8ClampedArray,
-  Int8Array,
-  Uint16Array,
-  Int16Array,
-  Uint32Array,
-  Int32Array,
-  Float32Array,
-  Float64Array,
-  BigUint64Array,
-  BigInt64Array,
-};
-
-export type TypedArray =
-  | Uint8Array
-  | Uint8ClampedArray
-  | Int8Array
-  | Uint16Array
-  | Int16Array
-  | Uint32Array
-  | Int32Array
-  | Float32Array
-  | Float64Array
-  | BigUint64Array
-  | BigInt64Array;
-
-export interface OverlayMask {
-  buffer: ArrayBuffer;
-  shape: [number, number];
-  channels: number;
-  arrayType: keyof typeof ARRAY_TYPES;
-}
+import type { OverlayMask } from "./types.js";
 
 const DATA_TYPES = {
   // < = little-endian, > = big-endian, | = host architecture
@@ -146,6 +111,6 @@ function parse(array: Uint8Array): OverlayMask {
 /**
  * Deserializes and parses a base64 encoded numpy array
  */
-function deserialize(compressedBase64Array: string): OverlayMask {
+export function deserialize(compressedBase64Array: string): OverlayMask {
   return parse(pako.inflate(Buffer.from(compressedBase64Array, "base64")));
 }

app/packages/looker/src/worker/decoders/tiff.ts

@@ -1,14 +1,25 @@
-import { getFetchFunction } from "@fiftyone/utilities";
-import { decode } from "decode-tiff";
-import type { OverlayMask } from "../../numpy";
-
-export default async (url: string) => {
-  const buffer: ArrayBuffer = await getFetchFunction()(
-    "GET",
-    url,
-    null,
-    "arrayBuffer"
-  );
-  const result = decode(buffer);
-  return {} as OverlayMask;
+import * as tiff from "geotiff";
+import type { OverlayMask } from "./types";
+
+export default async (blob: Blob) => {
+  const r = await tiff.fromBlob(blob);
+  const image = await r.getImage();
+  console.log(image.getHeight(), image.getWidth());
+  console.log(r);
+  console.log(image);
+
+  console.log();
+
+  const data = await image.readRGB();
+
+  if (!(data instanceof Uint8Array)) {
+    throw new Error("wrong");
+  }
+
+  return {
+    arrayType: "Uint8Array",
+    buffer: data.buffer,
+    shape: [4, 4],
+    channels: 2,
+  } as OverlayMask;
 };