Skip to content

Commit

Permalink
organizing
Browse files Browse the repository at this point in the history
  • Loading branch information
benjaminpkane committed Jan 2, 2025
1 parent b3c36de commit 19b3c7e
Show file tree
Hide file tree
Showing 23 changed files with 4,877 additions and 690 deletions.
6 changes: 6 additions & 0 deletions app/packages/looker/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,27 @@
"README.md"
],
"dependencies": {
"@jimp/tiff": "^0.22.12",
"@ungap/event-target": "^0.2.2",
"@xmldom/xmldom": "^0.8.6",
"copy-to-clipboard": "^3.3.1",
"decode-tiff": "^0.2.1",
"geotiff": "^2.1.3",
"immutable": "^4.0.0-rc.12",
"lodash": "^4.17.21",
"lru-cache": "^11.0.1",
"mime": "^2.5.2",
"monotone-convex-hull-2d": "^1.0.1",
"tiff": "^6.1.1",
"utif": "^3.1.0",
"utif2": "^4.1.0",
"uuid": "^8.3.2"
},
"devDependencies": {
"@rollup/plugin-inject": "^5.0.2",
"@types/color-string": "^1.5.0",
"@types/lru-cache": "^7.10.10",
"@types/utif": "^3.0.5",
"@types/uuid": "^8.3.0",
"buffer": "^6.0.3",
"prettier": "^2.7.1",
Expand Down
6 changes: 1 addition & 5 deletions app/packages/looker/src/numpy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ export const ARRAY_TYPES = {
Int32Array,
Float32Array,
Float64Array,
BigUint64Array,
BigInt64Array,
};

export type TypedArray =
Expand All @@ -30,9 +28,7 @@ export type TypedArray =
| Uint32Array
| Int32Array
| Float32Array
| Float64Array
| BigUint64Array
| BigInt64Array;
| Float64Array;

export interface OverlayMask {
buffer: ArrayBuffer;
Expand Down
39 changes: 11 additions & 28 deletions app/packages/looker/src/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@
* Copyright 2017-2024, Voxel51, Inc.
*/

import { BufferManager } from "@fiftyone/utilities";
import { ImaVidFramesController } from "./lookers/imavid/controller";
import { Overlay } from "./overlays/base";

import { AppError, COLOR_BY, Schema, Stage } from "@fiftyone/utilities";
import type {
AppError,
BufferManager,
COLOR_BY,
Schema,
Stage,
} from "@fiftyone/utilities";
import type { ImaVidFramesController } from "./lookers/imavid/controller";
import type { Overlay } from "./overlays/base";

export type Optional<T> = {
[P in keyof T]?: Optional<T[P]>;
Expand Down Expand Up @@ -70,27 +74,6 @@ export type OrthogrpahicProjectionMetadata = {
normal: [number, number, number];
};

export type GenericLabel = {
[labelKey: string]: {
[field: string]: unknown;
};
// todo: add other label types
};

export type Sample = {
metadata: {
width: number;
height: number;
mime_type?: string;
};
_id: string;
id: string;
filepath: string;
tags: string[];
_label_tags: string[];
_media_type: "image" | "video" | "point-cloud" | "3d";
} & GenericLabel;

export interface LabelData {
labelId: string;
field: string;
Expand Down Expand Up @@ -127,8 +110,8 @@ export type Action<State extends BaseState> = (
) => void;

export enum ControlEventKeyType {
HOLD,
KEY_DOWN,
HOLD = "HOLD",
KEY_DOWN = "KEY_DOWN",
}
export interface Control<State extends BaseState = BaseState> {
eventKeys?: string | string[];
Expand Down
9 changes: 1 addition & 8 deletions app/packages/looker/src/worker/decoders/canvas.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import type { OverlayMask } from "../../numpy";
import { enqueueFetch } from "../pooled-fetch";

const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
/**
Expand Down Expand Up @@ -34,13 +33,7 @@ const getPngcolorType = async (blob: Blob): Promise<number | undefined> => {
return colorType;
};

const decodeWithCanvas = async (url: string): Promise<OverlayMask> => {
const overlayImageFetchResponse = await enqueueFetch({
url,
options: { priority: "low" },
});
const blob = await overlayImageFetchResponse.blob();

const decodeWithCanvas = async (blob: Blob): Promise<OverlayMask> => {
let channels = 4;

if (blob.type === "image/png") {
Expand Down
16 changes: 9 additions & 7 deletions app/packages/looker/src/worker/decoders/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ import { DETECTION, DETECTIONS } from "@fiftyone/utilities";
import type { Coloring, CustomizeColor } from "../..";
import type { OverlayMask } from "../../numpy";
import type { Colorscale } from "../../state";
import { enqueueFetch } from "../pooled-fetch";
import { getOverlayFieldFromCls } from "../shared";
import decodeCanvas from "./canvas";
import decodeTiff from "./tiff";

type Decoder = (url: string) => Promise<OverlayMask>;
type Decoder = (blob: Blob) => Promise<OverlayMask>;

const IMAGE_DECODERS: { [key: string]: Decoder } = {
jpeg: decodeCanvas,
Expand All @@ -17,11 +18,6 @@ const IMAGE_DECODERS: { [key: string]: Decoder } = {
tiff: decodeTiff,
};

async function decodeImage(url: string) {
const extension = url.split(".").slice(-1)[0];
return IMAGE_DECODERS[extension](url);
}

export type IntermediateMask = {
data: OverlayMask;
image: ArrayBuffer;
Expand Down Expand Up @@ -110,7 +106,13 @@ const decode = async ({ field, label, ...params }: DecodeParameters) => {

// convert absolute file path to a URL that we can "fetch" from
const overlayImageUrl = getSampleSrc(source || label[overlayPathField]);
const overlayMask = await decodeImage(overlayImageUrl);
const overlayImageFetchResponse = await enqueueFetch({
url: overlayImageUrl,
options: { priority: "low" },
});
const blob = await overlayImageFetchResponse.blob();
const extension = overlayImageUrl.split(".").slice(-1)[0];
const overlayMask = await IMAGE_DECODERS[extension](blob);
const [overlayHeight, overlayWidth] = overlayMask.shape;

// set the `mask` property for this label
Expand Down
23 changes: 23 additions & 0 deletions app/packages/looker/src/worker/decoders/inline.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { deserialize } from "./numpy";
import type { InlineDecodeable, IntermediateMask } from "./types";

export default (inline: InlineDecodeable): IntermediateMask | null => {
let base64: string;
if (typeof inline === "string") {
base64 = inline;
} else if (typeof inline?.$binary?.base64 === "string") {
base64 = inline.$binary.base64;
}

if (!base64) {
return null;
}

const data = deserialize(base64);
const [height, width] = data.shape;

return {
data,
image: new ArrayBuffer(width * height * 4),
};
};
116 changes: 116 additions & 0 deletions app/packages/looker/src/worker/decoders/numpy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/**
* Copyright 2017-2024, Voxel51, Inc.
*/

import { Buffer } from "buffer";
import pako from "./pako.js";
import type { OverlayMask } from "./types.js";

const DATA_TYPES = {
// < = little-endian, > = big-endian, | = host architecture
// we assume hosts are little-endian (like x86) so big-endian types are only
// supported for 8-bit integers, where endianness doesn't matter
"|b1": Uint8Array,
"<b1": Uint8Array,
">b1": Uint8Array,
"|u1": Uint8Array,
"<u1": Uint8Array,
">u1": Uint8Array,
"|i1": Int8Array,
"<i1": Int8Array,
">i1": Int8Array,

"|u2": Uint16Array,
"<u2": Uint16Array,
"|i2": Int16Array,
"<i2": Int16Array,

"|u4": Uint32Array,
"<u4": Uint32Array,
"|i4": Int32Array,
"<i4": Int32Array,

"|u8": BigUint64Array,
"<u8": BigUint64Array,
"|i8": BigInt64Array,
"<i8": BigInt64Array,

"<f4": Float32Array,
"|f4": Float32Array,

"<f8": Float64Array,
"|f8": Float64Array,
};

/**
* Parses a uint16 (unsigned 16-bit integer) at a specified position in a
* Uint8Array
*/
function readUint16At(array: Uint8Array, index: number): number {
return array[index] + (array[index + 1] << 8);
}

/**
* Parses a string at a specified position in a Uint8Array
*/
function readStringAt(array: Uint8Array, start: number, end: number) {
return Array.from(array.slice(start, end))
.map((c) => String.fromCharCode(c))
.join("");
}

/**
* Parses a saved numpy array
*/
function parse(array: Uint8Array): OverlayMask {
const version = readUint16At(array, 6);
if (version !== 1) {
throw new Error(`Unsupported version: ${version}`);
}
const headerLength = readUint16At(array, 8);
const bodyIndex = 10 + headerLength;
const header = JSON.parse(
readStringAt(array, 10, bodyIndex)
.replace(/'/g, '"')
.replace(/\(/g, "[")
.replace(/\)/g, "]")
.replace(/True|False/g, (s) => s.toLowerCase())
.replace(/\s+/g, "")
.replace(/,}/, "}")
.replace(/,\]/, "]")
);

if (header.fortran_order) {
throw new Error(`Fortran order arrays are not supported"`);
}

const ArrayType = DATA_TYPES[header.descr];

if (!ArrayType) {
throw new Error(`Unsupported data type: "${header.descr}"`);
}
const rawData = array.slice(bodyIndex);

const typedData =
ArrayType === Uint8Array
? rawData
: new ArrayType(
rawData.buffer,
rawData.byteOffset,
rawData.byteLength / ArrayType.BYTES_PER_ELEMENT
);

return {
arrayType: typedData.constructor.name,
buffer: typedData.buffer,
channels: header.shape[2] ?? 1,
shape: [header.shape[0], header.shape[1]],
};
}

/**
* Deserializes and parses a base64 encoded numpy array
*/
export function deserialize(compressedBase64Array: string): OverlayMask {
return parse(pako.inflate(Buffer.from(compressedBase64Array, "base64")));
}
Loading

0 comments on commit 19b3c7e

Please sign in to comment.