From a6bdcbcd5d65e083b01e98d254088a621ed3a785 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 27 Dec 2024 11:06:35 -0600 Subject: [PATCH 1/6] don't process overlays if path is there but overlay not decoded --- app/packages/looker/src/processOverlays.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/app/packages/looker/src/processOverlays.ts b/app/packages/looker/src/processOverlays.ts index c31ee19695..789fbd84d1 100644 --- a/app/packages/looker/src/processOverlays.ts +++ b/app/packages/looker/src/processOverlays.ts @@ -4,6 +4,8 @@ import { CONTAINS, Overlay } from "./overlays/base"; import { ClassificationsOverlay } from "./overlays/classifications"; +import HeatmapOverlay from "./overlays/heatmap"; +import SegmentationOverlay from "./overlays/segmentation"; import { BaseState } from "./state"; import { rotate } from "./util"; @@ -30,6 +32,25 @@ const processOverlays = ( if (!(overlay.field && overlay.field in bins)) continue; + // todo: find a better approach / place for this. + // for instance, this won't work in detection overlay, where + // we might want the bounding boxes but masks might not have been loaded + if ( + overlay instanceof SegmentationOverlay && + overlay.label.mask_path && + !overlay.label.mask + ) { + continue; + } + + if ( + overlay instanceof HeatmapOverlay && + overlay.label.map_path && + !overlay.label.map + ) { + continue; + } + if (!overlay.isShown(state)) continue; bins[overlay.field].push(overlay); From f79d22dff36a013ca2e8d2071a5dd17e583a8154 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 1 Jan 2025 21:57:24 -0600 Subject: [PATCH 2/6] set targets to null during cleanup --- app/packages/looker/src/overlays/heatmap.ts | 1 + app/packages/looker/src/overlays/segmentation.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/app/packages/looker/src/overlays/heatmap.ts b/app/packages/looker/src/overlays/heatmap.ts index d8fb8909d5..720aa8af36 100644 --- a/app/packages/looker/src/overlays/heatmap.ts +++ b/app/packages/looker/src/overlays/heatmap.ts @@ -208,6 +208,7 @@ export default class HeatmapOverlay public cleanup(): void { this.label.map?.bitmap?.close(); + this.targets = null; } } diff --git a/app/packages/looker/src/overlays/segmentation.ts b/app/packages/looker/src/overlays/segmentation.ts index 04c2fc693b..f16894ca7e 100644 --- a/app/packages/looker/src/overlays/segmentation.ts +++ b/app/packages/looker/src/overlays/segmentation.ts @@ -263,6 +263,7 @@ export default class SegmentationOverlay public cleanup(): void { this.label.mask?.bitmap?.close(); + this.targets = null; } } From 26fd18294de7a984ebae8bbf114f551fa0dae860 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 1 Jan 2025 21:58:26 -0600 Subject: [PATCH 3/6] cleanup in destroy and not reconciliation func --- app/packages/looker/src/lookers/abstract.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 08991adba9..602e4bc7f0 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -293,11 +293,6 @@ export abstract class AbstractLooker< return; } - if (this.state.destroyed && this.sampleOverlays) { - // close all current overlays - this.pluckedOverlays.forEach((overlay) => overlay.cleanup?.()); - } - if ( !this.state.windowBBox || this.state.destroyed || @@ -609,6 +604,7 @@ export abstract class AbstractLooker< this.detach(); this.abortController.abort(); this.updater({ destroyed: true }); + this.sampleOverlays?.forEach((overlay) => overlay.cleanup?.()); } disable() { From ba294eb06648901ace1fbea5b69146c97c8fe9f5 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Wed, 1 Jan 2025 22:02:36 -0600 Subject: [PATCH 4/6] cancel animation in destroy --- app/packages/spotlight/src/createScrollReader.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/packages/spotlight/src/createScrollReader.ts b/app/packages/spotlight/src/createScrollReader.ts index d164f1b5ab..0a1abfbf39 100644 --- a/app/packages/spotlight/src/createScrollReader.ts +++ b/app/packages/spotlight/src/createScrollReader.ts @@ -9,6 +9,7 @@ export default function createScrollReader( render: (zooming: boolean, dispatchOffset?: boolean) => void, getScrollSpeedThreshold: () => number ) { + let animationFrameId: ReturnType; let destroyed = false; let prior: number; let scrolling = false; @@ -63,7 +64,7 @@ export default function createScrollReader( updateScrollStatus(); } - requestAnimationFrame(animate); + animationFrameId = requestAnimationFrame(animate); }; animate(); @@ -73,6 +74,14 @@ export default function createScrollReader( destroyed = true; element.removeEventListener("scroll", scroll); element.removeEventListener("scrollend", scrollEnd); + + if (timeout) { + clearTimeout(timeout); + } + + if (animationFrameId) { + cancelAnimationFrame(animationFrameId); + } }, zooming: () => zooming, }; From 4bced93495df714018eb92fca9e47fec2bc4a4bc Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 2 Jan 2025 11:29:45 -0600 Subject: [PATCH 5/6] reuse offscreen canvas --- app/packages/looker/src/worker/canvas-decoder.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/packages/looker/src/worker/canvas-decoder.ts b/app/packages/looker/src/worker/canvas-decoder.ts index 56749a6d2e..3af00d430a 100644 --- a/app/packages/looker/src/worker/canvas-decoder.ts +++ b/app/packages/looker/src/worker/canvas-decoder.ts @@ -1,5 +1,10 @@ import { OverlayMask } from "../numpy"; +const offScreenCanvas = new OffscreenCanvas(1, 1); +const offScreenCanvasCtx = offScreenCanvas.getContext("2d", { + willReadFrequently: true, +})!; + const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; /** * Reads the PNG's image header chunk to determine the color type. @@ -53,12 +58,14 @@ export const decodeWithCanvas = async (blob: Blob) => { const imageBitmap = await createImageBitmap(blob); const { width, height } = imageBitmap; - const canvas = new OffscreenCanvas(width, height); - const ctx = canvas.getContext("2d")!; - ctx.drawImage(imageBitmap, 0, 0); + offScreenCanvas.width = width; + offScreenCanvas.height = height; + + offScreenCanvasCtx.drawImage(imageBitmap, 0, 0); + imageBitmap.close(); - const imageData = ctx.getImageData(0, 0, width, height); + const imageData = offScreenCanvasCtx.getImageData(0, 0, width, height); if (channels === 1) { // get rid of the G, B, and A channels, new buffer will be 1/4 the size From e9e749e1ce811f95c73f4e720d32a72f95c351ff Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 2 Jan 2025 12:50:13 -0600 Subject: [PATCH 6/6] modify buffer in place --- .../looker/src/worker/canvas-decoder.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/packages/looker/src/worker/canvas-decoder.ts b/app/packages/looker/src/worker/canvas-decoder.ts index 3af00d430a..d0ed29f69f 100644 --- a/app/packages/looker/src/worker/canvas-decoder.ts +++ b/app/packages/looker/src/worker/canvas-decoder.ts @@ -69,11 +69,20 @@ export const decodeWithCanvas = async (blob: Blob) => { if (channels === 1) { // get rid of the G, B, and A channels, new buffer will be 1/4 the size - const data = new Uint8ClampedArray(width * height); - for (let i = 0; i < data.length; i++) { - data[i] = imageData.data[i * 4]; + const rawBuffer = imageData.data; + const totalPixels = width * height; + + let read = 0; + let write = 0; + + while (write < totalPixels) { + rawBuffer[write++] = rawBuffer[read]; + // skip "G,B,A" + read += 4; } - imageData.data.set(data); + + const grayScaleData = rawBuffer.slice(0, totalPixels); + rawBuffer.set(grayScaleData); } return {