From 9c3cb74832267105f280cab1131a8fce9516fd65 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 14:38:45 -0600 Subject: [PATCH 1/8] add detections fields to additional media fields --- fiftyone/server/metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fiftyone/server/metadata.py b/fiftyone/server/metadata.py index 6992447e4a..b5303731b9 100644 --- a/fiftyone/server/metadata.py +++ b/fiftyone/server/metadata.py @@ -31,6 +31,8 @@ logger = logging.getLogger(__name__) _ADDITIONAL_MEDIA_FIELDS = { + fol.Detection: "mask_path", + fol.Detections: "mask_path", fol.Heatmap: "map_path", fol.Segmentation: "mask_path", OrthographicProjectionMetadata: "filepath", From bd7d2de1c942c640ddaf961892aeac8d81ecc8cc Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 14:40:36 -0600 Subject: [PATCH 2/8] use pydash.get instead of _deep_get --- fiftyone/server/metadata.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/fiftyone/server/metadata.py b/fiftyone/server/metadata.py index b5303731b9..bfab94e403 100644 --- a/fiftyone/server/metadata.py +++ b/fiftyone/server/metadata.py @@ -12,6 +12,7 @@ import typing as t from functools import reduce +from pydash import get import asyncio import aiofiles @@ -415,7 +416,7 @@ def _create_media_urls( media_urls = [] for field in media_fields: - path = _deep_get(sample, field) + path = get(sample, field) if path not in cache: cache[path] = path @@ -452,15 +453,3 @@ def _get_additional_media_fields( additional.append(f"{field_name}.{subfield_name}") return opm_field, additional - - -def _deep_get(sample, keys, default=None): - """ - Get a value from a nested dictionary by specifying keys delimited by '.', - similar to lodash's ``_.get()``. - """ - return reduce( - lambda d, key: d.get(key, default) if isinstance(d, dict) else default, - keys.split("."), - sample, - ) From d0462b9f7026dcc8574ca514ba0fbbc731e30722 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 14:44:45 -0600 Subject: [PATCH 3/8] use detections fields in media urls creation, too --- fiftyone/server/metadata.py | 38 ++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/fiftyone/server/metadata.py b/fiftyone/server/metadata.py index bfab94e403..47d3a41bf8 100644 --- a/fiftyone/server/metadata.py +++ b/fiftyone/server/metadata.py @@ -71,7 +71,11 @@ async def get_metadata( filepath = sample["filepath"] metadata = sample.get("metadata", None) - opm_field, additional_fields = _get_additional_media_fields(collection) + ( + opm_field, + detections_fields, + additional_fields, + ) = _get_additional_media_fields(collection) filepath_result, filepath_source, urls = _create_media_urls( collection, @@ -80,6 +84,7 @@ async def get_metadata( url_cache, additional_fields=additional_fields, opm_field=opm_field, + detections_fields=detections_fields, ) if filepath_result is not None: filepath = filepath_result @@ -392,6 +397,7 @@ def _create_media_urls( cache: t.Dict, additional_fields: t.Optional[t.List[str]] = None, opm_field: t.Optional[str] = None, + detections_fields: t.Optional[t.List[str]] = None, ) -> t.Dict[str, str]: filepath_source = None media_fields = collection.app_config.media_fields.copy() @@ -399,6 +405,23 @@ def _create_media_urls( if additional_fields is not None: media_fields.extend(additional_fields) + if detections_fields is not None: + for field in detections_fields: + detections = get(sample, field) + + if not detections: + continue + + detections_list = get(detections, "detections") + + if not detections_list or len(detections_list) == 0: + continue + + len_detections = len(detections_list) + + for i in range(len_detections): + media_fields.append(f"{field}.detections[{i}].mask_path") + if ( sample_media_type == fom.POINT_CLOUD or sample_media_type == fom.THREE_D @@ -438,6 +461,8 @@ def _get_additional_media_fields( ) -> t.List[str]: additional = [] opm_field = None + detections_fields = None + for cls, subfield_name in _ADDITIONAL_MEDIA_FIELDS.items(): for field_name, field in collection.get_field_schema( flat=True @@ -450,6 +475,13 @@ def _get_additional_media_fields( if cls == OrthographicProjectionMetadata: opm_field = field_name - additional.append(f"{field_name}.{subfield_name}") + if cls == fol.Detections: + if detections_fields is None: + detections_fields = [field_name] + else: + detections_fields.append(field_name) + + else: + additional.append(f"{field_name}.{subfield_name}") - return opm_field, additional + return opm_field, detections_fields, additional From 89d18538603cd08bd62bf6a8ab7eb45e89a1e7ab Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 14:45:28 -0600 Subject: [PATCH 4/8] add support for collection overlay types in disk decoder --- .../looker/src/worker/disk-overlay-decoder.ts | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/app/packages/looker/src/worker/disk-overlay-decoder.ts b/app/packages/looker/src/worker/disk-overlay-decoder.ts index c5ac65acd1..a7a793b4a9 100644 --- a/app/packages/looker/src/worker/disk-overlay-decoder.ts +++ b/app/packages/looker/src/worker/disk-overlay-decoder.ts @@ -25,12 +25,17 @@ export const decodeOverlayOnDisk = async ( sources: { [path: string]: string }, cls: string, maskPathDecodingPromises: Promise[] = [], - maskTargetsBuffers: ArrayBuffer[] = [] + maskTargetsBuffers: ArrayBuffer[] = [], + overlayCollectionProcessingParams: + | { idx: number; cls: string } + | undefined = undefined ) => { // handle all list types here - if (cls === DETECTIONS) { + if (cls === DETECTIONS && label.detections) { const promises: Promise[] = []; - for (const detection of label.detections) { + + for (let i = 0; i < label.detections.length; i++) { + const detection = label.detections[i]; promises.push( decodeOverlayOnDisk( field, @@ -38,10 +43,11 @@ export const decodeOverlayOnDisk = async ( coloring, customizeColorSetting, colorscale, - {}, + sources, DETECTION, maskPathDecodingPromises, - maskTargetsBuffers + maskTargetsBuffers, + { idx: i, cls: DETECTIONS } ) ); } @@ -74,6 +80,17 @@ export const decodeOverlayOnDisk = async ( return; } + // if we have an explicit source defined from sample.urls, use that + // otherwise, use the path field from the label + let source = sources[`${field}.${overlayPathField}`]; + + if (typeof overlayCollectionProcessingParams !== "undefined") { + source = + sources[ + `${field}.${overlayCollectionProcessingParams.cls}[${overlayCollectionProcessingParams.idx}].${overlayPathField}` + ]; + } + // convert absolute file path to a URL that we can "fetch" from const overlayImageUrl = getSampleSrc( sources[`${field}.${overlayPathField}`] || label[overlayPathField] From d2038b0f549c0593b7dafd07d5fa293b1121cf96 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 14:55:09 -0600 Subject: [PATCH 5/8] return if no path --- fiftyone/server/metadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fiftyone/server/metadata.py b/fiftyone/server/metadata.py index 47d3a41bf8..060b95a241 100644 --- a/fiftyone/server/metadata.py +++ b/fiftyone/server/metadata.py @@ -441,6 +441,9 @@ def _create_media_urls( for field in media_fields: path = get(sample, field) + if not path: + continue + if path not in cache: cache[path] = path From cd8eaee5f2d8bfc480d6fa91380b61a81d6d2fff Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 15:00:35 -0600 Subject: [PATCH 6/8] fix src bug --- app/packages/looker/src/worker/disk-overlay-decoder.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/packages/looker/src/worker/disk-overlay-decoder.ts b/app/packages/looker/src/worker/disk-overlay-decoder.ts index a7a793b4a9..c3aaea0074 100644 --- a/app/packages/looker/src/worker/disk-overlay-decoder.ts +++ b/app/packages/looker/src/worker/disk-overlay-decoder.ts @@ -92,9 +92,7 @@ export const decodeOverlayOnDisk = async ( } // convert absolute file path to a URL that we can "fetch" from - const overlayImageUrl = getSampleSrc( - sources[`${field}.${overlayPathField}`] || label[overlayPathField] - ); + const overlayImageUrl = getSampleSrc(source || label[overlayPathField]); const urlTokens = overlayImageUrl.split("?"); let baseUrl = overlayImageUrl; From 5d6f683968065de6ac920554be308b40cb91592b Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 15:31:02 -0600 Subject: [PATCH 7/8] add clarification comment for sources --- app/packages/looker/src/worker/disk-overlay-decoder.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/packages/looker/src/worker/disk-overlay-decoder.ts b/app/packages/looker/src/worker/disk-overlay-decoder.ts index c3aaea0074..73b5d02b6c 100644 --- a/app/packages/looker/src/worker/disk-overlay-decoder.ts +++ b/app/packages/looker/src/worker/disk-overlay-decoder.ts @@ -85,9 +85,13 @@ export const decodeOverlayOnDisk = async ( let source = sources[`${field}.${overlayPathField}`]; if (typeof overlayCollectionProcessingParams !== "undefined") { + // example: for detections, we need to access the source from the parent label + // like: if field is "prediction_masks", we're trying to get "predictiion_masks.detections[INDEX].mask" source = sources[ - `${field}.${overlayCollectionProcessingParams.cls}[${overlayCollectionProcessingParams.idx}].${overlayPathField}` + `${field}.${overlayCollectionProcessingParams.cls.toLocaleLowerCase()}[${ + overlayCollectionProcessingParams.idx + }].${overlayPathField}` ]; } From bc44e3ea5221c1f898ce1d3ae41accbd3c39acc9 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 17 Dec 2024 15:31:35 -0600 Subject: [PATCH 8/8] don't get rid of query params if source is defined --- app/packages/looker/src/worker/disk-overlay-decoder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/looker/src/worker/disk-overlay-decoder.ts b/app/packages/looker/src/worker/disk-overlay-decoder.ts index 73b5d02b6c..2948831061 100644 --- a/app/packages/looker/src/worker/disk-overlay-decoder.ts +++ b/app/packages/looker/src/worker/disk-overlay-decoder.ts @@ -102,7 +102,7 @@ export const decodeOverlayOnDisk = async ( let baseUrl = overlayImageUrl; // remove query params if not local URL - if (!urlTokens.at(1)?.startsWith("filepath=")) { + if (!urlTokens.at(1)?.startsWith("filepath=") && !source) { baseUrl = overlayImageUrl.split("?")[0]; }