Skip to content

Commit

Permalink
fe work
Browse files Browse the repository at this point in the history
  • Loading branch information
@benjaminpkane
benjaminpkane committed Dec 23, 2024
1 parent 685d855 commit b3c36de
Show file tree
Hide file tree
Showing 11 changed files with 248 additions and 687 deletions.
1 change: 1 addition & 0 deletions app/packages/looker/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"@ungap/event-target": "^0.2.2",
"@xmldom/xmldom": "^0.8.6",
"copy-to-clipboard": "^3.3.1",
"decode-tiff": "^0.2.1",
"immutable": "^4.0.0-rc.12",
"lodash": "^4.17.21",
"lru-cache": "^11.0.1",
Expand Down
4 changes: 2 additions & 2 deletions app/packages/looker/src/util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ export const createWorker = (
worker.addEventListener(
"error",
(error) => {
dispatchEvent("error", error);
dispatchEvent?.("error", error);
},
signal
);
Expand All @@ -476,7 +476,7 @@ export const createWorker = (
const error = !ERRORS[data.error.cls]
? new Error(data.error.message)
: new ERRORS[data.error.cls](data.error.data, data.error.message);
dispatchEvent("error", new ErrorEvent("error", { error }));
dispatchEvent?.("error", new ErrorEvent("error", { error }));
}
},
signal
Expand Down
29 changes: 21 additions & 8 deletions ...kages/looker/src/worker/canvas-decoder.ts → ...ages/looker/src/worker/decoders/canvas.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { OverlayMask } from "../numpy";
import type { OverlayMask } from "../../numpy";
import { enqueueFetch } from "../pooled-fetch";

const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
/**
Expand All @@ -9,9 +10,13 @@ const getPngcolorType = async (blob: Blob): Promise<number | undefined> => {
// https://www.w3.org/TR/2003/REC-PNG-20031110/#11IHDR

// PNG signature is 8 bytes
// IHDR (image header): length(4 bytes), chunk type(4 bytes), then data(13 bytes)
// IHDR (image header): length(4 bytes), chunk type(4 bytes),
// then data(13 bytes)
//
// data layout of IHDR: width(4), height(4), bit depth(1), color type(1), ...
// color type is at offset: 8(signature) + 4(length) + 4(chunk type) + 8(width+height) + 1(bit depth)
//
// color type is at offset:
// 8(signature) + 4(length) + 4(chunk type) + 8(width+height) + 1(bit depth)
// = 8 + 4 + 4 + 8 + 1 = 25 (0-based index)

const header = new Uint8Array(await blob.slice(0, 26).arrayBuffer());
Expand All @@ -29,8 +34,14 @@ const getPngcolorType = async (blob: Blob): Promise<number | undefined> => {
return colorType;
};

export const decodeWithCanvas = async (blob: Blob) => {
let channels: number = 4;
const decodeWithCanvas = async (url: string): Promise<OverlayMask> => {
const overlayImageFetchResponse = await enqueueFetch({
url,
options: { priority: "low" },
});
const blob = await overlayImageFetchResponse.blob();

let channels = 4;

if (blob.type === "image/png") {
const colorType = await getPngcolorType(blob);
Expand All @@ -54,7 +65,7 @@ export const decodeWithCanvas = async (blob: Blob) => {
const { width, height } = imageBitmap;

const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext("2d")!;
const ctx = canvas.getContext("2d");
ctx.drawImage(imageBitmap, 0, 0);
imageBitmap.close();

Expand All @@ -70,9 +81,11 @@ export const decodeWithCanvas = async (blob: Blob) => {
}

return {
arrayType: "Uint8ClampedArray",
buffer: imageData.data.buffer,
channels,
arrayType: "Uint8ClampedArray",
shape: [height, width],
} as OverlayMask;
};
};

export default decodeWithCanvas;
119 changes: 61 additions & 58 deletions ...r/src/worker/disk-overlay-decoder.test.ts → .../looker/src/worker/decoders/index.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,48 @@
import { getSampleSrc } from "@fiftyone/state/src/recoil/utils";
import { DETECTIONS, HEATMAP } from "@fiftyone/utilities";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { Coloring, CustomizeColor } from "..";
import { LabelMask } from "../overlays/base";
import type { Colorscale } from "../state";
import { decodeWithCanvas } from "./canvas-decoder";
import { decodeOverlayOnDisk, IntermediateMask } from "./disk-overlay-decoder";
import { enqueueFetch } from "./pooled-fetch";
import type { IntermediateMask } from ".";
import decode from ".";
import type { Coloring, CustomizeColor } from "../..";
import type { OverlayMask } from "../../numpy";
import type { LabelMask } from "../../overlays/base";
import type { Colorscale } from "../../state";
import { enqueueFetch } from "../pooled-fetch";
import canvas from "./canvas";

vi.mock("@fiftyone/state/src/recoil/utils", () => ({
getSampleSrc: vi.fn(),
}));

vi.mock("./pooled-fetch", () => ({
vi.mock("../pooled-fetch", () => ({
enqueueFetch: vi.fn(),
}));

vi.mock("./canvas-decoder", () => ({
vi.mock("./canvas", () => ({
decodeWithCanvas: vi.fn(),
}));

const createOverlayMask = (shape: [number, number]): OverlayMask => {
return {
arrayType: "Uint8Array",
buffer: new ArrayBuffer(shape[0] * shape[1]),
channels: 1,
shape,
};
};

const COLORING = {} as Coloring;
const COLOR_SCALE = {} as Colorscale;
const CUSTOMIZE_COLOR_SETTING: CustomizeColor[] = [];
const SOURCES = {};

const PARAMS = {
coloring: COLORING,
colorscale: COLOR_SCALE,
customizeColorSetting: CUSTOMIZE_COLOR_SETTING,
sources: SOURCES,
};

type MaskUnion = (IntermediateMask & LabelMask) | null;

describe("decodeOverlayOnDisk", () => {
Expand All @@ -38,16 +56,13 @@ describe("decodeOverlayOnDisk", () => {
const cls = "Segmentation";
const maskPathDecodingPromises: Promise<void>[] = [];

await decodeOverlayOnDisk(
await decode({
cls,
field,
label,
COLORING,
CUSTOMIZE_COLOR_SETTING,
COLOR_SCALE,
SOURCES,
cls,
maskPathDecodingPromises
);
maskPathDecodingPromises,
...PARAMS,
});

expect(label.mask).toBeDefined();
expect(enqueueFetch).not.toHaveBeenCalled();
Expand All @@ -61,31 +76,28 @@ describe("decodeOverlayOnDisk", () => {

const sampleSrcUrl = "http://example.com/path/to/mask";
const mockBlob = new Blob(["mock data"], { type: "image/png" });
const overlayMask = { shape: [100, 200] };
const overlayMask = createOverlayMask([100, 200]);

vi.mocked(getSampleSrc).mockReturnValue(sampleSrcUrl);
vi.mocked(enqueueFetch).mockResolvedValue({
blob: () => Promise.resolve(mockBlob),
} as Response);
vi.mocked(decodeWithCanvas).mockResolvedValue(overlayMask);
vi.mocked(canvas).mockResolvedValue(overlayMask);

await decodeOverlayOnDisk(
await decode({
cls,
field,
label,
COLORING,
CUSTOMIZE_COLOR_SETTING,
COLOR_SCALE,
SOURCES,
cls,
maskPathDecodingPromises
);
maskPathDecodingPromises,
...PARAMS,
});

expect(getSampleSrc).toHaveBeenCalledWith("/path/to/mask");
expect(enqueueFetch).toHaveBeenCalledWith({
url: sampleSrcUrl,
options: { priority: "low" },
});
expect(decodeWithCanvas).toHaveBeenCalledWith(mockBlob);
expect(canvas).toHaveBeenCalledWith(mockBlob);
expect(label.mask).toBeDefined();
expect(label.mask.data).toBe(overlayMask);
expect(label.mask.image).toBeInstanceOf(ArrayBuffer);
Expand All @@ -100,28 +112,25 @@ describe("decodeOverlayOnDisk", () => {

const sampleSrcUrl = "http://example.com/path/to/map";
const mockBlob = new Blob(["mock data"], { type: "image/png" });
const overlayMask = { shape: [100, 200] };
const overlayMask = createOverlayMask([100, 200]);

vi.mocked(getSampleSrc).mockReturnValue(sampleSrcUrl);
vi.mocked(decodeWithCanvas).mockResolvedValue(overlayMask);
vi.mocked(canvas).mockResolvedValue(overlayMask);

await decodeOverlayOnDisk(
await decode({
cls,
field,
label,
COLORING,
CUSTOMIZE_COLOR_SETTING,
COLOR_SCALE,
SOURCES,
cls,
maskPathDecodingPromises
);
maskPathDecodingPromises,
...PARAMS,
});

expect(getSampleSrc).toHaveBeenCalledWith("/path/to/map");
expect(enqueueFetch).toHaveBeenCalledWith({
url: sampleSrcUrl,
options: { priority: "low" },
});
expect(decodeWithCanvas).toHaveBeenCalledWith(mockBlob);
expect(canvas).toHaveBeenCalledWith(mockBlob);
expect(label.map).toBeDefined();
expect(label.map.data).toBe(overlayMask);
expect(label.map.image).toBeInstanceOf(ArrayBuffer);
Expand All @@ -141,26 +150,23 @@ describe("decodeOverlayOnDisk", () => {

const sampleSrcUrl1 = "http://example.com/path/to/mask1";
const sampleSrcUrl2 = "http://example.com/path/to/mask2";
const overlayMask1 = { shape: [50, 50] };
const overlayMask2 = { shape: [60, 60] };
const overlayMask1 = createOverlayMask([50, 50]);
const overlayMask2 = createOverlayMask([60, 60]);

vi.mocked(getSampleSrc)
.mockReturnValueOnce(sampleSrcUrl1)
.mockReturnValueOnce(sampleSrcUrl2);
vi.mocked(decodeWithCanvas)
vi.mocked(canvas)
.mockResolvedValueOnce(overlayMask1)
.mockResolvedValueOnce(overlayMask2);

await decodeOverlayOnDisk(
await decode({
cls,
field,
label,
COLORING,
CUSTOMIZE_COLOR_SETTING,
COLOR_SCALE,
SOURCES,
cls,
maskPathDecodingPromises
);
maskPathDecodingPromises,
...PARAMS,
});

await Promise.all(maskPathDecodingPromises);

Expand All @@ -183,23 +189,20 @@ describe("decodeOverlayOnDisk", () => {
vi.mocked(getSampleSrc).mockReturnValue(sampleSrcUrl);
vi.mocked(enqueueFetch).mockRejectedValue(new Error("Fetch failed"));

await decodeOverlayOnDisk(
await decode({
cls,
field,
label,
COLORING,
CUSTOMIZE_COLOR_SETTING,
COLOR_SCALE,
SOURCES,
cls,
maskPathDecodingPromises
);
maskPathDecodingPromises,
...PARAMS,
});

expect(getSampleSrc).toHaveBeenCalledWith("/path/to/mask");
expect(enqueueFetch).toHaveBeenCalledWith({
url: sampleSrcUrl,
options: { priority: "low" },
});
expect(decodeWithCanvas).not.toHaveBeenCalled();
expect(canvas).not.toHaveBeenCalled();
expect(label.mask).toBeNull();
});
});
Loading

0 comments on commit b3c36de

Please sign in to comment.