From ff35f5bab0ce204f7f97e7b274c7f00c43957f8e Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 3 Dec 2024 15:56:52 -0600 Subject: [PATCH 01/37] add default_visibility_labels to dataset app config --- fiftyone/core/odm/dataset.py | 3 +++ fiftyone/server/query.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/fiftyone/core/odm/dataset.py b/fiftyone/core/odm/dataset.py index 62cfb1ccbe..816dfc2851 100644 --- a/fiftyone/core/odm/dataset.py +++ b/fiftyone/core/odm/dataset.py @@ -515,6 +515,8 @@ class DatasetAppConfig(EmbeddedDocument): Args: color_scheme (None): an optional :class:`ColorScheme` for the dataset + default_visibility_labels (None): config with include and exclude lists + of labels to render by default in the App disable_frame_filtering (False): whether to disable frame filtering for video datasets in the App's grid view grid_media_field ("filepath"): the default sample field from which to @@ -546,6 +548,7 @@ class DatasetAppConfig(EmbeddedDocument): meta = {"strict": False} color_scheme = EmbeddedDocumentField(ColorScheme, default=None) + default_visibility_labels = DictField(default=None) disable_frame_filtering = BooleanField(default=None) dynamic_groups_target_frame_rate = IntField(default=30) grid_media_field = StringField(default="filepath") diff --git a/fiftyone/server/query.py b/fiftyone/server/query.py index 3960796c0a..31271df737 100644 --- a/fiftyone/server/query.py +++ b/fiftyone/server/query.py @@ -59,6 +59,12 @@ DATASET_FILTER_STAGE = [{"$match": DATASET_FILTER[0]}] +@gql.type +class FieldVisibilityConfig: + include: t.Optional[t.List[str]] + exclude: t.Optional[t.List[str]] + + @gql.type class Group: name: str @@ -210,6 +216,7 @@ class NamedKeypointSkeleton(KeypointSkeleton): @gql.type class DatasetAppConfig: color_scheme: t.Optional[ColorScheme] + default_visibility_labels: t.Optional[FieldVisibilityConfig] disable_frame_filtering: t.Optional[bool] = None dynamic_groups_target_frame_rate: int = 30 grid_media_field: str = "filepath" From 0c8bd80e2000bc4336f9ec477abb0c9cb6264a79 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 3 Dec 2024 16:26:44 -0600 Subject: [PATCH 02/37] update gql for new app config --- .../__generated__/DatasetPageQuery.graphql.ts | 31 +++++++++++++++-- .../datasetAppConfigFragment.graphql.ts | 33 +++++++++++++++++-- .../src/fragments/datasetAppConfigFragment.ts | 4 +++ .../__generated__/datasetQuery.graphql.ts | 31 +++++++++++++++-- app/packages/state/src/recoil/types.ts | 6 ++++ app/schema.graphql | 6 ++++ 6 files changed, 103 insertions(+), 8 deletions(-) diff --git a/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts b/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts index f4e92d4510..a17b25a05c 100644 --- a/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts +++ b/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<> + * @generated SignedSource<> * @lightSyntaxTransform * @nogrep */ @@ -847,6 +847,31 @@ return { "plural": false, "selections": [ (v24/*: any*/), + { + "alias": null, + "args": null, + "concreteType": "FieldVisibilityConfig", + "kind": "LinkedField", + "name": "defaultVisibilityLabels", + "plural": false, + "selections": [ + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "include", + "storageKey": null + }, + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "exclude", + "storageKey": null + } + ], + "storageKey": null + }, (v25/*: any*/), { "alias": null, @@ -1452,12 +1477,12 @@ return { ] }, "params": { - "cacheID": "f99a6a7bf7511890fdd5db148152135a", + "cacheID": "6075b4fbbfbf557c858ae76d346cc506", "id": null, "metadata": {}, "name": "DatasetPageQuery", "operationKind": "query", - "text": "query DatasetPageQuery(\n $count: Int\n $cursor: String\n $name: String!\n $extendedView: BSONArray!\n $savedViewSlug: String\n $search: String = \"\"\n $view: BSONArray!\n) {\n config {\n colorBy\n colorPool\n colorscale\n multicolorKeypoints\n showSkeletons\n }\n colorscale\n dataset(name: $name, view: $extendedView, savedViewSlug: $savedViewSlug) {\n name\n defaultGroupSlice\n appConfig {\n colorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n }\n }\n ...datasetFragment\n id\n }\n ...NavFragment\n ...savedViewsFragment\n ...configFragment\n ...stageDefinitionsFragment\n ...viewSchemaFragment\n}\n\nfragment Analytics on Query {\n context\n dev\n doNotTrack\n uid\n version\n}\n\nfragment NavDatasets on Query {\n datasets(search: $search, first: $count, after: $cursor) {\n total\n edges {\n cursor\n node {\n name\n id\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment NavFragment on Query {\n ...Analytics\n ...NavDatasets\n}\n\nfragment colorSchemeFragment on ColorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n}\n\nfragment configFragment on Query {\n config {\n colorBy\n colorPool\n colorscale\n disableFrameFiltering\n gridZoom\n enableQueryPerformance\n defaultQueryPerformance\n loopVideos\n mediaFallback\n multicolorKeypoints\n notebookHeight\n plugins\n showConfidence\n showIndex\n showLabel\n showSkeletons\n showTooltip\n theme\n timezone\n useFrameNumber\n }\n colorscale\n}\n\nfragment datasetAppConfigFragment on DatasetAppConfig {\n colorScheme {\n ...colorSchemeFragment\n id\n }\n disableFrameFiltering\n dynamicGroupsTargetFrameRate\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n plugins\n}\n\nfragment datasetFragment on Dataset {\n createdAt\n datasetId\n groupField\n id\n info\n lastLoadedAt\n mediaType\n name\n parentMediaType\n version\n appConfig {\n ...datasetAppConfigFragment\n }\n brainMethods {\n key\n version\n timestamp\n viewStages\n config {\n cls\n embeddingsField\n method\n patchesField\n supportsPrompts\n type\n maxK\n supportsLeastSimilarity\n }\n }\n defaultMaskTargets {\n target\n value\n }\n defaultSkeleton {\n labels\n edges\n }\n evaluations {\n key\n version\n timestamp\n viewStages\n config {\n cls\n predField\n gtField\n }\n }\n groupMediaTypes {\n name\n mediaType\n }\n maskTargets {\n name\n targets {\n target\n value\n }\n }\n skeletons {\n name\n labels\n edges\n }\n ...estimatedCountsFragment\n ...frameFieldsFragment\n ...groupSliceFragment\n ...indexesFragment\n ...mediaFieldsFragment\n ...mediaTypeFragment\n ...sampleFieldsFragment\n ...sidebarGroupsFragment\n ...viewFragment\n}\n\nfragment estimatedCountsFragment on Dataset {\n estimatedFrameCount\n estimatedSampleCount\n}\n\nfragment frameFieldsFragment on Dataset {\n frameFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment groupSliceFragment on Dataset {\n defaultGroupSlice\n}\n\nfragment indexesFragment on Dataset {\n frameIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n sampleIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n}\n\nfragment mediaFieldsFragment on Dataset {\n name\n appConfig {\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n }\n sampleFields {\n path\n }\n}\n\nfragment mediaTypeFragment on Dataset {\n mediaType\n}\n\nfragment sampleFieldsFragment on Dataset {\n sampleFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment savedViewsFragment on Query {\n savedViews(datasetName: $name) {\n id\n datasetId\n name\n slug\n description\n color\n viewStages\n createdAt\n lastModifiedAt\n lastLoadedAt\n }\n}\n\nfragment sidebarGroupsFragment on Dataset {\n datasetId\n appConfig {\n sidebarGroups {\n expanded\n paths\n name\n }\n }\n ...frameFieldsFragment\n ...sampleFieldsFragment\n}\n\nfragment stageDefinitionsFragment on Query {\n stageDefinitions {\n name\n params {\n name\n type\n default\n placeholder\n }\n }\n}\n\nfragment viewFragment on Dataset {\n stages(slug: $savedViewSlug, view: $view)\n viewCls\n viewName\n}\n\nfragment viewSchemaFragment on Query {\n schemaForViewStages(datasetName: $name, viewStages: $view) {\n fieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n frameFieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n }\n}\n" + "text": "query DatasetPageQuery(\n $count: Int\n $cursor: String\n $name: String!\n $extendedView: BSONArray!\n $savedViewSlug: String\n $search: String = \"\"\n $view: BSONArray!\n) {\n config {\n colorBy\n colorPool\n colorscale\n multicolorKeypoints\n showSkeletons\n }\n colorscale\n dataset(name: $name, view: $extendedView, savedViewSlug: $savedViewSlug) {\n name\n defaultGroupSlice\n appConfig {\n colorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n }\n }\n ...datasetFragment\n id\n }\n ...NavFragment\n ...savedViewsFragment\n ...configFragment\n ...stageDefinitionsFragment\n ...viewSchemaFragment\n}\n\nfragment Analytics on Query {\n context\n dev\n doNotTrack\n uid\n version\n}\n\nfragment NavDatasets on Query {\n datasets(search: $search, first: $count, after: $cursor) {\n total\n edges {\n cursor\n node {\n name\n id\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment NavFragment on Query {\n ...Analytics\n ...NavDatasets\n}\n\nfragment colorSchemeFragment on ColorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n}\n\nfragment configFragment on Query {\n config {\n colorBy\n colorPool\n colorscale\n disableFrameFiltering\n gridZoom\n enableQueryPerformance\n defaultQueryPerformance\n loopVideos\n mediaFallback\n multicolorKeypoints\n notebookHeight\n plugins\n showConfidence\n showIndex\n showLabel\n showSkeletons\n showTooltip\n theme\n timezone\n useFrameNumber\n }\n colorscale\n}\n\nfragment datasetAppConfigFragment on DatasetAppConfig {\n colorScheme {\n ...colorSchemeFragment\n id\n }\n defaultVisibilityLabels {\n include\n exclude\n }\n disableFrameFiltering\n dynamicGroupsTargetFrameRate\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n plugins\n}\n\nfragment datasetFragment on Dataset {\n createdAt\n datasetId\n groupField\n id\n info\n lastLoadedAt\n mediaType\n name\n parentMediaType\n version\n appConfig {\n ...datasetAppConfigFragment\n }\n brainMethods {\n key\n version\n timestamp\n viewStages\n config {\n cls\n embeddingsField\n method\n patchesField\n supportsPrompts\n type\n maxK\n supportsLeastSimilarity\n }\n }\n defaultMaskTargets {\n target\n value\n }\n defaultSkeleton {\n labels\n edges\n }\n evaluations {\n key\n version\n timestamp\n viewStages\n config {\n cls\n predField\n gtField\n }\n }\n groupMediaTypes {\n name\n mediaType\n }\n maskTargets {\n name\n targets {\n target\n value\n }\n }\n skeletons {\n name\n labels\n edges\n }\n ...estimatedCountsFragment\n ...frameFieldsFragment\n ...groupSliceFragment\n ...indexesFragment\n ...mediaFieldsFragment\n ...mediaTypeFragment\n ...sampleFieldsFragment\n ...sidebarGroupsFragment\n ...viewFragment\n}\n\nfragment estimatedCountsFragment on Dataset {\n estimatedFrameCount\n estimatedSampleCount\n}\n\nfragment frameFieldsFragment on Dataset {\n frameFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment groupSliceFragment on Dataset {\n defaultGroupSlice\n}\n\nfragment indexesFragment on Dataset {\n frameIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n sampleIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n}\n\nfragment mediaFieldsFragment on Dataset {\n name\n appConfig {\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n }\n sampleFields {\n path\n }\n}\n\nfragment mediaTypeFragment on Dataset {\n mediaType\n}\n\nfragment sampleFieldsFragment on Dataset {\n sampleFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment savedViewsFragment on Query {\n savedViews(datasetName: $name) {\n id\n datasetId\n name\n slug\n description\n color\n viewStages\n createdAt\n lastModifiedAt\n lastLoadedAt\n }\n}\n\nfragment sidebarGroupsFragment on Dataset {\n datasetId\n appConfig {\n sidebarGroups {\n expanded\n paths\n name\n }\n }\n ...frameFieldsFragment\n ...sampleFieldsFragment\n}\n\nfragment stageDefinitionsFragment on Query {\n stageDefinitions {\n name\n params {\n name\n type\n default\n placeholder\n }\n }\n}\n\nfragment viewFragment on Dataset {\n stages(slug: $savedViewSlug, view: $view)\n viewCls\n viewName\n}\n\nfragment viewSchemaFragment on Query {\n schemaForViewStages(datasetName: $name, viewStages: $view) {\n fieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n frameFieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n }\n}\n" } }; })(); diff --git a/app/packages/relay/src/fragments/__generated__/datasetAppConfigFragment.graphql.ts b/app/packages/relay/src/fragments/__generated__/datasetAppConfigFragment.graphql.ts index 56bd9e042a..19d0043bcf 100644 --- a/app/packages/relay/src/fragments/__generated__/datasetAppConfigFragment.graphql.ts +++ b/app/packages/relay/src/fragments/__generated__/datasetAppConfigFragment.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<<67543879e3e2987632bd17a53759bb4f>> + * @generated SignedSource<<6d98ce427e7903abab6b8c839b2a8f9a>> * @lightSyntaxTransform * @nogrep */ @@ -14,6 +14,10 @@ export type datasetAppConfigFragment$data = { readonly colorScheme: { readonly " $fragmentSpreads": FragmentRefs<"colorSchemeFragment">; } | null; + readonly defaultVisibilityLabels: { + readonly exclude: ReadonlyArray | null; + readonly include: ReadonlyArray | null; + } | null; readonly disableFrameFiltering: boolean | null; readonly dynamicGroupsTargetFrameRate: number; readonly gridMediaField: string; @@ -50,6 +54,31 @@ const node: ReaderFragment = { ], "storageKey": null }, + { + "alias": null, + "args": null, + "concreteType": "FieldVisibilityConfig", + "kind": "LinkedField", + "name": "defaultVisibilityLabels", + "plural": false, + "selections": [ + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "include", + "storageKey": null + }, + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "exclude", + "storageKey": null + } + ], + "storageKey": null + }, { "alias": null, "args": null, @@ -104,6 +133,6 @@ const node: ReaderFragment = { "abstractKey": null }; -(node as any).hash = "ee329c6ed9452236fb9ea680bea410e3"; +(node as any).hash = "575efb48b8c550944ea2223802d85123"; export default node; diff --git a/app/packages/relay/src/fragments/datasetAppConfigFragment.ts b/app/packages/relay/src/fragments/datasetAppConfigFragment.ts index 5a40822015..7bdc948cdd 100644 --- a/app/packages/relay/src/fragments/datasetAppConfigFragment.ts +++ b/app/packages/relay/src/fragments/datasetAppConfigFragment.ts @@ -5,6 +5,10 @@ export default graphql` colorScheme { ...colorSchemeFragment } + defaultVisibilityLabels { + include + exclude + } disableFrameFiltering dynamicGroupsTargetFrameRate gridMediaField diff --git a/app/packages/relay/src/queries/__generated__/datasetQuery.graphql.ts b/app/packages/relay/src/queries/__generated__/datasetQuery.graphql.ts index eecf98bdb9..b9f840724a 100644 --- a/app/packages/relay/src/queries/__generated__/datasetQuery.graphql.ts +++ b/app/packages/relay/src/queries/__generated__/datasetQuery.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<<1d71c12e715c42191d7686db91155645>> + * @generated SignedSource<> * @lightSyntaxTransform * @nogrep */ @@ -864,6 +864,31 @@ return { "plural": false, "selections": [ (v20/*: any*/), + { + "alias": null, + "args": null, + "concreteType": "FieldVisibilityConfig", + "kind": "LinkedField", + "name": "defaultVisibilityLabels", + "plural": false, + "selections": [ + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "include", + "storageKey": null + }, + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "exclude", + "storageKey": null + } + ], + "storageKey": null + }, (v23/*: any*/), { "alias": null, @@ -1337,12 +1362,12 @@ return { ] }, "params": { - "cacheID": "d5cd5a5f9cb4d8fc0c14dd15336df4fb", + "cacheID": "065424bbe58df165ebc333d83f82bba6", "id": null, "metadata": {}, "name": "datasetQuery", "operationKind": "query", - "text": "query datasetQuery(\n $extendedView: BSONArray!\n $name: String!\n $savedViewSlug: String\n $view: BSONArray!\n $workspaceSlug: String\n) {\n config {\n colorBy\n colorPool\n colorscale\n multicolorKeypoints\n showSkeletons\n }\n dataset(name: $name, view: $extendedView, savedViewSlug: $savedViewSlug) {\n name\n defaultGroupSlice\n viewName\n savedViewSlug\n appConfig {\n colorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n labelTags {\n fieldColor\n valueColors {\n value\n color\n }\n }\n fields {\n colorByAttribute\n fieldColor\n path\n maskTargetsColors {\n intTarget\n color\n }\n valueColors {\n color\n value\n }\n }\n }\n }\n workspace(slug: $workspaceSlug) {\n id\n child\n slug\n }\n ...datasetFragment\n id\n }\n ...savedViewsFragment\n ...configFragment\n ...stageDefinitionsFragment\n ...viewSchemaFragment\n}\n\nfragment colorSchemeFragment on ColorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n}\n\nfragment configFragment on Query {\n config {\n colorBy\n colorPool\n colorscale\n disableFrameFiltering\n gridZoom\n enableQueryPerformance\n defaultQueryPerformance\n loopVideos\n mediaFallback\n multicolorKeypoints\n notebookHeight\n plugins\n showConfidence\n showIndex\n showLabel\n showSkeletons\n showTooltip\n theme\n timezone\n useFrameNumber\n }\n colorscale\n}\n\nfragment datasetAppConfigFragment on DatasetAppConfig {\n colorScheme {\n ...colorSchemeFragment\n id\n }\n disableFrameFiltering\n dynamicGroupsTargetFrameRate\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n plugins\n}\n\nfragment datasetFragment on Dataset {\n createdAt\n datasetId\n groupField\n id\n info\n lastLoadedAt\n mediaType\n name\n parentMediaType\n version\n appConfig {\n ...datasetAppConfigFragment\n }\n brainMethods {\n key\n version\n timestamp\n viewStages\n config {\n cls\n embeddingsField\n method\n patchesField\n supportsPrompts\n type\n maxK\n supportsLeastSimilarity\n }\n }\n defaultMaskTargets {\n target\n value\n }\n defaultSkeleton {\n labels\n edges\n }\n evaluations {\n key\n version\n timestamp\n viewStages\n config {\n cls\n predField\n gtField\n }\n }\n groupMediaTypes {\n name\n mediaType\n }\n maskTargets {\n name\n targets {\n target\n value\n }\n }\n skeletons {\n name\n labels\n edges\n }\n ...estimatedCountsFragment\n ...frameFieldsFragment\n ...groupSliceFragment\n ...indexesFragment\n ...mediaFieldsFragment\n ...mediaTypeFragment\n ...sampleFieldsFragment\n ...sidebarGroupsFragment\n ...viewFragment\n}\n\nfragment estimatedCountsFragment on Dataset {\n estimatedFrameCount\n estimatedSampleCount\n}\n\nfragment frameFieldsFragment on Dataset {\n frameFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment groupSliceFragment on Dataset {\n defaultGroupSlice\n}\n\nfragment indexesFragment on Dataset {\n frameIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n sampleIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n}\n\nfragment mediaFieldsFragment on Dataset {\n name\n appConfig {\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n }\n sampleFields {\n path\n }\n}\n\nfragment mediaTypeFragment on Dataset {\n mediaType\n}\n\nfragment sampleFieldsFragment on Dataset {\n sampleFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment savedViewsFragment on Query {\n savedViews(datasetName: $name) {\n id\n datasetId\n name\n slug\n description\n color\n viewStages\n createdAt\n lastModifiedAt\n lastLoadedAt\n }\n}\n\nfragment sidebarGroupsFragment on Dataset {\n datasetId\n appConfig {\n sidebarGroups {\n expanded\n paths\n name\n }\n }\n ...frameFieldsFragment\n ...sampleFieldsFragment\n}\n\nfragment stageDefinitionsFragment on Query {\n stageDefinitions {\n name\n params {\n name\n type\n default\n placeholder\n }\n }\n}\n\nfragment viewFragment on Dataset {\n stages(slug: $savedViewSlug, view: $view)\n viewCls\n viewName\n}\n\nfragment viewSchemaFragment on Query {\n schemaForViewStages(datasetName: $name, viewStages: $view) {\n fieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n frameFieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n }\n}\n" + "text": "query datasetQuery(\n $extendedView: BSONArray!\n $name: String!\n $savedViewSlug: String\n $view: BSONArray!\n $workspaceSlug: String\n) {\n config {\n colorBy\n colorPool\n colorscale\n multicolorKeypoints\n showSkeletons\n }\n dataset(name: $name, view: $extendedView, savedViewSlug: $savedViewSlug) {\n name\n defaultGroupSlice\n viewName\n savedViewSlug\n appConfig {\n colorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n labelTags {\n fieldColor\n valueColors {\n value\n color\n }\n }\n fields {\n colorByAttribute\n fieldColor\n path\n maskTargetsColors {\n intTarget\n color\n }\n valueColors {\n color\n value\n }\n }\n }\n }\n workspace(slug: $workspaceSlug) {\n id\n child\n slug\n }\n ...datasetFragment\n id\n }\n ...savedViewsFragment\n ...configFragment\n ...stageDefinitionsFragment\n ...viewSchemaFragment\n}\n\nfragment colorSchemeFragment on ColorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n}\n\nfragment configFragment on Query {\n config {\n colorBy\n colorPool\n colorscale\n disableFrameFiltering\n gridZoom\n enableQueryPerformance\n defaultQueryPerformance\n loopVideos\n mediaFallback\n multicolorKeypoints\n notebookHeight\n plugins\n showConfidence\n showIndex\n showLabel\n showSkeletons\n showTooltip\n theme\n timezone\n useFrameNumber\n }\n colorscale\n}\n\nfragment datasetAppConfigFragment on DatasetAppConfig {\n colorScheme {\n ...colorSchemeFragment\n id\n }\n defaultVisibilityLabels {\n include\n exclude\n }\n disableFrameFiltering\n dynamicGroupsTargetFrameRate\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n plugins\n}\n\nfragment datasetFragment on Dataset {\n createdAt\n datasetId\n groupField\n id\n info\n lastLoadedAt\n mediaType\n name\n parentMediaType\n version\n appConfig {\n ...datasetAppConfigFragment\n }\n brainMethods {\n key\n version\n timestamp\n viewStages\n config {\n cls\n embeddingsField\n method\n patchesField\n supportsPrompts\n type\n maxK\n supportsLeastSimilarity\n }\n }\n defaultMaskTargets {\n target\n value\n }\n defaultSkeleton {\n labels\n edges\n }\n evaluations {\n key\n version\n timestamp\n viewStages\n config {\n cls\n predField\n gtField\n }\n }\n groupMediaTypes {\n name\n mediaType\n }\n maskTargets {\n name\n targets {\n target\n value\n }\n }\n skeletons {\n name\n labels\n edges\n }\n ...estimatedCountsFragment\n ...frameFieldsFragment\n ...groupSliceFragment\n ...indexesFragment\n ...mediaFieldsFragment\n ...mediaTypeFragment\n ...sampleFieldsFragment\n ...sidebarGroupsFragment\n ...viewFragment\n}\n\nfragment estimatedCountsFragment on Dataset {\n estimatedFrameCount\n estimatedSampleCount\n}\n\nfragment frameFieldsFragment on Dataset {\n frameFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment groupSliceFragment on Dataset {\n defaultGroupSlice\n}\n\nfragment indexesFragment on Dataset {\n frameIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n sampleIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n}\n\nfragment mediaFieldsFragment on Dataset {\n name\n appConfig {\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n }\n sampleFields {\n path\n }\n}\n\nfragment mediaTypeFragment on Dataset {\n mediaType\n}\n\nfragment sampleFieldsFragment on Dataset {\n sampleFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment savedViewsFragment on Query {\n savedViews(datasetName: $name) {\n id\n datasetId\n name\n slug\n description\n color\n viewStages\n createdAt\n lastModifiedAt\n lastLoadedAt\n }\n}\n\nfragment sidebarGroupsFragment on Dataset {\n datasetId\n appConfig {\n sidebarGroups {\n expanded\n paths\n name\n }\n }\n ...frameFieldsFragment\n ...sampleFieldsFragment\n}\n\nfragment stageDefinitionsFragment on Query {\n stageDefinitions {\n name\n params {\n name\n type\n default\n placeholder\n }\n }\n}\n\nfragment viewFragment on Dataset {\n stages(slug: $savedViewSlug, view: $view)\n viewCls\n viewName\n}\n\nfragment viewSchemaFragment on Query {\n schemaForViewStages(datasetName: $name, viewStages: $view) {\n fieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n frameFieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n }\n}\n" } }; })(); diff --git a/app/packages/state/src/recoil/types.ts b/app/packages/state/src/recoil/types.ts index 1013cdf52e..61445b8a10 100644 --- a/app/packages/state/src/recoil/types.ts +++ b/app/packages/state/src/recoil/types.ts @@ -20,6 +20,11 @@ export namespace State { * configuration for that plugin. */ export type PluginConfig = { [pluginName: string]: object }; + + export type DefaultVisibilityLabelsConfig = { + include?: string[]; + exclude?: string[]; + }; export interface Config { colorPool: string[]; customizedColors: CustomizeColorInput[]; @@ -119,6 +124,7 @@ export namespace State { } export interface DatasetAppConfig { + defaultVisibilityLabels?: DefaultVisibilityLabelsConfig; dynamicGroupsTargetFrameRate: number; gridMediaField?: string; modalMediaField?: string; diff --git a/app/schema.graphql b/app/schema.graphql index 844bf714d4..4a7eef66d9 100644 --- a/app/schema.graphql +++ b/app/schema.graphql @@ -247,6 +247,7 @@ type Dataset { type DatasetAppConfig { colorScheme: ColorScheme + defaultVisibilityLabels: FieldVisibilityConfig disableFrameFiltering: Boolean dynamicGroupsTargetFrameRate: Int! gridMediaField: String! @@ -329,6 +330,11 @@ input ExtendedViewForm { slice: String = null } +type FieldVisibilityConfig { + include: [String!] + exclude: [String!] +} + type FloatAggregation implements Aggregation { path: String! count: Int! From 89c48817b8bde1ceeaaf3f9815d5f4d81385016e Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 3 Dec 2024 16:53:15 -0600 Subject: [PATCH 03/37] add selector for default visibility labels --- app/packages/state/src/recoil/selectors.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/packages/state/src/recoil/selectors.ts b/app/packages/state/src/recoil/selectors.ts index adb8a05698..ddd239d934 100644 --- a/app/packages/state/src/recoil/selectors.ts +++ b/app/packages/state/src/recoil/selectors.ts @@ -142,6 +142,15 @@ export const datasetAppConfig = graphQLSyncFragmentAtom< } ); +export const defaultVisibilityLabels = + selector({ + key: "defaultVisibilityLabels", + get: ({ get }) => { + return get(datasetAppConfig) + .defaultVisibilityLabels as State.DefaultVisibilityLabelsConfig | null; + }, + }); + export const disableFrameFiltering = selector({ key: "disableFrameFiltering", get: ({ get }) => { From f2083cfccdd72aa305465923717deb7e47e5d6fe Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 5 Dec 2024 13:06:46 -0600 Subject: [PATCH 04/37] add function to get dense labels list --- app/packages/utilities/src/schema.ts | 37 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/app/packages/utilities/src/schema.ts b/app/packages/utilities/src/schema.ts index 188ab1eb37..030f56e163 100644 --- a/app/packages/utilities/src/schema.ts +++ b/app/packages/utilities/src/schema.ts @@ -1,3 +1,16 @@ +export const DETECTION_EMBEDDED_DOC_TYPE = "fiftyone.core.labels.Detection"; +export const DETECTIONS_EMBEDDED_DOC_TYPE = "fiftyone.core.labels.Detections"; +export const SEGMENTATION_EMBEDDED_DOC_TYPE = + "fiftyone.core.labels.Segmentation"; +export const HEATMAP_EMBEDDED_DOC_TYPE = "fiftyone.core.labels.Heatmap"; + +export const DENSE_LABEL_EMBEDDED_DOC_TYPES = [ + DETECTION_EMBEDDED_DOC_TYPE, + DETECTIONS_EMBEDDED_DOC_TYPE, + SEGMENTATION_EMBEDDED_DOC_TYPE, + HEATMAP_EMBEDDED_DOC_TYPE, +]; + export interface Field { ftype: string; dbField: string | null; @@ -54,21 +67,29 @@ export function getCls(fieldPath: string, schema: Schema): string | undefined { export function getFieldsWithEmbeddedDocType( schema: Schema, - embeddedDocType: string + embeddedDocType: string | string[], + shouldRecurse = true ): Field[] { const result: Field[] = []; function recurse(schema: Schema) { for (const field of Object.values(schema ?? {})) { - if (field.embeddedDocType === embeddedDocType) { + if (Array.isArray(embeddedDocType)) { + if (embeddedDocType.includes(field.embeddedDocType)) { + result.push(field); + } + } else if (field.embeddedDocType === embeddedDocType) { result.push(field); } if (field.fields) { - recurse(field.fields); + if (shouldRecurse) { + recurse(field.fields); + } } } } + // need to call it once regardless of shouldRecurse recurse(schema); return result; } @@ -91,3 +112,13 @@ export function doesSchemaContainEmbeddedDocType( return recurse(schema); } + +export function getDenseLabelNames(schema: Schema): string[] { + const denseLabels = getFieldsWithEmbeddedDocType( + schema, + DENSE_LABEL_EMBEDDED_DOC_TYPES, + false + ); + + return denseLabels.map((label) => label.name); +} From 42da2fb03f8873332126f7c30985b4b76a47db2a Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 5 Dec 2024 17:23:44 -0600 Subject: [PATCH 05/37] new hook for setting sidebar labels --- app/packages/core/src/components/Dataset.tsx | 7 ++- app/packages/core/src/hooks/index.ts | 1 + .../hooks/useOptimizeDenseLabelsLoading.ts | 53 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts diff --git a/app/packages/core/src/components/Dataset.tsx b/app/packages/core/src/components/Dataset.tsx index d533cbda6f..cb6c165564 100644 --- a/app/packages/core/src/components/Dataset.tsx +++ b/app/packages/core/src/components/Dataset.tsx @@ -1,14 +1,15 @@ +import { useTrackEvent } from "@fiftyone/analytics"; import { subscribe } from "@fiftyone/relay"; import { isModalActive } from "@fiftyone/state"; import React, { useEffect } from "react"; import { useRecoilValue } from "recoil"; import styled from "styled-components"; +import { useOptimizeDenseLabelsLoading } from "../hooks"; import ColorModal from "./ColorModal/ColorModal"; import { activeColorEntry } from "./ColorModal/state"; +import EventTracker from "./EventTracker"; import Modal from "./Modal"; import SamplesContainer from "./SamplesContainer"; -import EventTracker from "./EventTracker"; -import { useTrackEvent } from "@fiftyone/analytics"; const Container = styled.div` height: 100%; @@ -41,6 +42,8 @@ function Dataset() { }); }, []); + useOptimizeDenseLabelsLoading(); + return ( <> diff --git a/app/packages/core/src/hooks/index.ts b/app/packages/core/src/hooks/index.ts index 7e1d3cc750..15c9c5c16f 100644 --- a/app/packages/core/src/hooks/index.ts +++ b/app/packages/core/src/hooks/index.ts @@ -1 +1,2 @@ +export * from "./useOptimizeDenseLabelsLoading"; export { default as useRefetchableSavedViews } from "./useRefetchableSavedViews"; diff --git a/app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts b/app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts new file mode 100644 index 0000000000..85d7a68146 --- /dev/null +++ b/app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts @@ -0,0 +1,53 @@ +import { + activeFields, + defaultVisibilityLabels, + fieldSchema, + State, +} from "@fiftyone/state"; +import { getDenseLabelNames } from "@fiftyone/utilities"; +import { useEffect, useMemo } from "react"; +import { useRecoilState, useRecoilValue } from "recoil"; + +export const useOptimizeDenseLabelsLoading = () => { + const defaultVisibleLabels = useRecoilValue(defaultVisibilityLabels); + const [alreadyActiveFields, setActiveFields] = useRecoilState( + activeFields({ modal: false }) + ); + + const schema = useRecoilValue(fieldSchema({ space: State.SPACE.SAMPLE })); + + const denseLabels = useMemo(() => getDenseLabelNames(schema), [schema]); + + useEffect(() => { + if (!denseLabels) { + return; + } + + // if no user defined defaults, turn off all dense labels + const starting = new Set(alreadyActiveFields); + + // if (!defaultVisibleLabels?.include && !defaultVisibleLabels?.exclude) { + // for now, exlude all dense labels + if (true) { + const filteredActiveFields = new Set( + [...starting].filter((x) => !denseLabels.includes(x)) + ); + setActiveFields([...filteredActiveFields]); + } + return; // temporary testing + + // get a list of all dense labels + + // userDefinedDefaults = set(include) - set(exclude) + const includeList = new Set(defaultVisibleLabels.include ?? []); + const excludeList = new Set(defaultVisibleLabels.exclude ?? []); + + const userDefinedDefaults = new Set( + [...includeList].filter((x) => !excludeList.has(x)) + ); + + // merge with already active fields + + // setActiveFields(defaultVisibleLabels); + }, [denseLabels, defaultVisibilityLabels]); +}; From 6f05c814bb1c8cd9ede4cdd5dfff3850be80c3af Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 5 Dec 2024 18:49:52 -0600 Subject: [PATCH 06/37] update _activeFields to respect default labels from app config (pending frames) --- app/packages/core/src/components/Dataset.tsx | 3 - app/packages/core/src/hooks/index.ts | 1 - .../hooks/useOptimizeDenseLabelsLoading.ts | 53 --------------- app/packages/state/src/recoil/schema.ts | 64 ++++++++++++++++++- 4 files changed, 63 insertions(+), 58 deletions(-) delete mode 100644 app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts diff --git a/app/packages/core/src/components/Dataset.tsx b/app/packages/core/src/components/Dataset.tsx index cb6c165564..613ea31e96 100644 --- a/app/packages/core/src/components/Dataset.tsx +++ b/app/packages/core/src/components/Dataset.tsx @@ -4,7 +4,6 @@ import { isModalActive } from "@fiftyone/state"; import React, { useEffect } from "react"; import { useRecoilValue } from "recoil"; import styled from "styled-components"; -import { useOptimizeDenseLabelsLoading } from "../hooks"; import ColorModal from "./ColorModal/ColorModal"; import { activeColorEntry } from "./ColorModal/state"; import EventTracker from "./EventTracker"; @@ -42,8 +41,6 @@ function Dataset() { }); }, []); - useOptimizeDenseLabelsLoading(); - return ( <> diff --git a/app/packages/core/src/hooks/index.ts b/app/packages/core/src/hooks/index.ts index 15c9c5c16f..7e1d3cc750 100644 --- a/app/packages/core/src/hooks/index.ts +++ b/app/packages/core/src/hooks/index.ts @@ -1,2 +1 @@ -export * from "./useOptimizeDenseLabelsLoading"; export { default as useRefetchableSavedViews } from "./useRefetchableSavedViews"; diff --git a/app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts b/app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts deleted file mode 100644 index 85d7a68146..0000000000 --- a/app/packages/core/src/hooks/useOptimizeDenseLabelsLoading.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - activeFields, - defaultVisibilityLabels, - fieldSchema, - State, -} from "@fiftyone/state"; -import { getDenseLabelNames } from "@fiftyone/utilities"; -import { useEffect, useMemo } from "react"; -import { useRecoilState, useRecoilValue } from "recoil"; - -export const useOptimizeDenseLabelsLoading = () => { - const defaultVisibleLabels = useRecoilValue(defaultVisibilityLabels); - const [alreadyActiveFields, setActiveFields] = useRecoilState( - activeFields({ modal: false }) - ); - - const schema = useRecoilValue(fieldSchema({ space: State.SPACE.SAMPLE })); - - const denseLabels = useMemo(() => getDenseLabelNames(schema), [schema]); - - useEffect(() => { - if (!denseLabels) { - return; - } - - // if no user defined defaults, turn off all dense labels - const starting = new Set(alreadyActiveFields); - - // if (!defaultVisibleLabels?.include && !defaultVisibleLabels?.exclude) { - // for now, exlude all dense labels - if (true) { - const filteredActiveFields = new Set( - [...starting].filter((x) => !denseLabels.includes(x)) - ); - setActiveFields([...filteredActiveFields]); - } - return; // temporary testing - - // get a list of all dense labels - - // userDefinedDefaults = set(include) - set(exclude) - const includeList = new Set(defaultVisibleLabels.include ?? []); - const excludeList = new Set(defaultVisibleLabels.exclude ?? []); - - const userDefinedDefaults = new Set( - [...includeList].filter((x) => !excludeList.has(x)) - ); - - // merge with already active fields - - // setActiveFields(defaultVisibleLabels); - }, [denseLabels, defaultVisibilityLabels]); -}; diff --git a/app/packages/state/src/recoil/schema.ts b/app/packages/state/src/recoil/schema.ts index d98504595c..6f9f4e865f 100644 --- a/app/packages/state/src/recoil/schema.ts +++ b/app/packages/state/src/recoil/schema.ts @@ -23,6 +23,7 @@ import { StrictField, VALID_NUMERIC_TYPES, VALID_PRIMITIVE_TYPES, + getDenseLabelNames, meetsFieldType, withPath, } from "@fiftyone/utilities"; @@ -31,6 +32,7 @@ import * as atoms from "./atoms"; import { dataset as datasetAtom } from "./dataset"; import { activeModalSample } from "./groups"; import { labelPathsSetExpanded } from "./labels"; +import { defaultVisibilityLabels } from "./selectors"; import { State } from "./types"; import { getLabelFields } from "./utils"; @@ -320,6 +322,66 @@ export const dbPath = selectorFamily({ }, }); +export const defaultLabels = selectorFamily({ + key: "defaultLabels", + get: + ({ space }) => + ({ get }) => { + const schema = get(fieldSchema({ space })); + const denseLabels = getDenseLabelNames(schema); + const allLabels = get(labelFields({ space })); + + const defaultVisibleLabelsConfig = get(defaultVisibilityLabels); + + if ( + !defaultVisibleLabelsConfig?.include && + !defaultVisibleLabelsConfig?.exclude + ) { + // todo: frames + return allLabels.filter((label) => !denseLabels.includes(label)); + } + + if ( + defaultVisibleLabelsConfig.include && + !defaultVisibleLabelsConfig.exclude + ) { + return allLabels.filter((label) => + defaultVisibleLabelsConfig.include.includes(label) + ); + } + + if ( + !defaultVisibleLabelsConfig.include && + defaultVisibleLabelsConfig.exclude + ) { + return allLabels.filter( + (label) => !defaultVisibleLabelsConfig.exclude.includes(label) + ); + } + + // is in both include and exclude + const includeList = new Set(defaultVisibleLabelsConfig.include); + const excludeList = new Set(defaultVisibleLabelsConfig.exclude); + // resolved = set(include) - set(exclude) + const resolved = new Set( + [...includeList].filter((x) => !excludeList.has(x)) + ); + return allLabels.filter((label) => resolved.has(label)); + + // todo: frames + if (space) { + return space === State.SPACE.FRAME + ? getLabelFields(get(atoms.frameFields), "frames.") + : getLabelFields(get(atoms.sampleFields)); + } + + return [ + ...getLabelFields(get(atoms.sampleFields)), + ...getLabelFields(get(atoms.frameFields), "frames."), + ]; + }, +}); + export const labelFields = selectorFamily({ key: "labelFields", get: @@ -513,7 +575,7 @@ export const activeFields = selectorFamily({ ({ modal }) => ({ get }) => { return filterPaths( - get(_activeFields({ modal })) || get(labelFields({})), + get(_activeFields({ modal })) || get(defaultLabels({})), buildSchema(get(atoms.sampleFields), get(atoms.frameFields)) ); }, From cab1d369f5741eef787601b70c8e49d21dc9eb45 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 5 Dec 2024 19:11:44 -0600 Subject: [PATCH 07/37] move color resolver to another module --- app/packages/looker/src/worker/index.ts | 49 +---------------- .../looker/src/worker/resolve-color.ts | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 47 deletions(-) create mode 100644 app/packages/looker/src/worker/resolve-color.ts diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index bf6cc966fc..71a95d5cf4 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -30,6 +30,7 @@ import { import { DeserializerFactory } from "./deserializer"; import { decodeOverlayOnDisk } from "./disk-overlay-decoder"; import { PainterFactory } from "./painter"; +import colorResolve from "./resolve-color"; import { getOverlayFieldFromCls, mapId } from "./shared"; import { process3DLabels } from "./threed-label-processor"; @@ -41,53 +42,7 @@ interface ResolveColor { type ResolveColorMethod = ReaderMethod & ResolveColor; -const [requestColor, resolveColor] = ((): [ - (pool: string[], seed: number, key: string | number) => Promise, - (result: ResolveColor) => void -] => { - const cache = {}; - const requests = {}; - const promises = {}; - - return [ - (pool, seed, key) => { - if (!(seed in cache)) { - cache[seed] = {}; - } - - const colors = cache[seed]; - - if (!(key in colors)) { - if (!(seed in requests)) { - requests[seed] = {}; - promises[seed] = {}; - } - - const seedRequests = requests[seed]; - const seedPromises = promises[seed]; - - if (!(key in seedRequests)) { - seedPromises[key] = new Promise((resolve) => { - seedRequests[key] = resolve; - postMessage({ - method: "requestColor", - key, - seed, - pool, - }); - }); - } - - return seedPromises[key]; - } - - return Promise.resolve(colors[key]); - }, - ({ key, seed, color }) => { - requests[seed][key](color); - }, - ]; -})(); +const [requestColor, resolveColor] = colorResolve(); const painterFactory = PainterFactory(requestColor); diff --git a/app/packages/looker/src/worker/resolve-color.ts b/app/packages/looker/src/worker/resolve-color.ts new file mode 100644 index 0000000000..aabb02c60b --- /dev/null +++ b/app/packages/looker/src/worker/resolve-color.ts @@ -0,0 +1,53 @@ +export interface ResolveColor { + key: string | number; + seed: number; + color: string; +} + +export default (): [ + (pool: string[], seed: number, key: string | number) => Promise, + (result: ResolveColor) => void +] => { + const cache = {}; + const requests = {}; + const promises = {}; + + return [ + (pool, seed, key) => { + if (!(seed in cache)) { + cache[seed] = {}; + } + + const colors = cache[seed]; + + if (!(key in colors)) { + if (!(seed in requests)) { + requests[seed] = {}; + promises[seed] = {}; + } + + const seedRequests = requests[seed]; + const seedPromises = promises[seed]; + + if (!(key in seedRequests)) { + seedPromises[key] = new Promise((resolve) => { + seedRequests[key] = resolve; + postMessage({ + method: "requestColor", + key, + seed, + pool, + }); + }); + } + + return seedPromises[key]; + } + + return Promise.resolve(colors[key]); + }, + ({ key, seed, color }) => { + requests[seed][key](color); + }, + ]; +}; From 9c057a7342efb4db551258068c7b0d11a6f265fe Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 5 Dec 2024 19:14:26 -0600 Subject: [PATCH 08/37] remove unused indexed png decoder code --- .../worker/tests/indexed-png-decoder.test.ts | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 app/packages/looker/src/worker/tests/indexed-png-decoder.test.ts diff --git a/app/packages/looker/src/worker/tests/indexed-png-decoder.test.ts b/app/packages/looker/src/worker/tests/indexed-png-decoder.test.ts deleted file mode 100644 index b190ca141b..0000000000 --- a/app/packages/looker/src/worker/tests/indexed-png-decoder.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - ColorPalette, - ValidPngBitDepth, - indexedPngBufferToRgb, -} from "../indexed-png-decoder"; - -describe("convertIndexedPngToRgb", () => { - const palette: ColorPalette = [ - [0, 0, 0], - [1, 1, 1], - [2, 2, 2], - [3, 3, 3], - ]; - - it("works with 1-bit depth", () => { - const inputData = new Uint8Array([0b10000000]); - const bitDepth: ValidPngBitDepth = 1; - // only two colors possible with 1 bit depth - // for 0b10000000, only first bit has value, so the first index color (1, 1, 1) is chosen - // other bits get 0 indexed color (0, 0, 0) - const expectedResult = new Uint8Array([ - 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - - const result = indexedPngBufferToRgb(inputData, bitDepth, palette); - expect(result).toEqual(expectedResult); - }); - - it("works with 2-bit depth", () => { - const inputData = new Uint8Array([0b11100000]); - const bitDepth: ValidPngBitDepth = 2; - // four colors possible with 2 bit depth - // first two bits = 11 = 3, so the third index color (3, 3, 3) is chosen - // second two bits = 10 = 2, so the second color is chosen - // other two-two bits (00, 00) translate to (0,0,0), (0,0,0) - const expectedResult = new Uint8Array([3, 3, 3, 2, 2, 2, 0, 0, 0, 0, 0, 0]); - - const result = indexedPngBufferToRgb(inputData, bitDepth, palette); - expect(result).toEqual(expectedResult); - }); - - it("works with 4-bit depth", () => { - const inputData = new Uint8Array([0b00110010]); - const bitDepth: ValidPngBitDepth = 4; - // 0011 = 3, so the third index color (3, 3, 3) is chosen - // 0010 = 2, so the second index color is chosen - const expectedResult = new Uint8Array([3, 3, 3, 2, 2, 2]); - - const result = indexedPngBufferToRgb(inputData, bitDepth, palette); - expect(result).toEqual(expectedResult); - }); - - it("works with 8-bit depth", () => { - const inputData = new Uint8Array([0b00000010, 0b00000011]); - const bitDepth: ValidPngBitDepth = 8; - // 00000010 = 2, so the second index color (2, 2, 2) is chosen - // 00000011 = 3, so the third index color (3, 3, 3) is chosen - const expectedResult = new Uint8Array([2, 2, 2, 3, 3, 3]); - - const result = indexedPngBufferToRgb(inputData, bitDepth, palette); - expect(result).toEqual(expectedResult); - }); -}); From 6d97b2dee9679c2b91452e0e4d0b5abaa8b5f007 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 20 Dec 2024 14:19:34 -0600 Subject: [PATCH 09/37] add .env to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index bb25c9979e..f352c3dae5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ dist/ coverage.xml .coverage.* pyvenv.cfg + +.env \ No newline at end of file From 12098947efbdea5d5bc3985646fe88564e4167e2 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 20 Dec 2024 15:12:14 -0600 Subject: [PATCH 10/37] don't process overlays if path is there but overlay not decoded --- app/packages/looker/src/processOverlays.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/packages/looker/src/processOverlays.ts b/app/packages/looker/src/processOverlays.ts index f02ffd6db4..a99dc5939a 100644 --- a/app/packages/looker/src/processOverlays.ts +++ b/app/packages/looker/src/processOverlays.ts @@ -30,6 +30,15 @@ const processOverlays = ( continue; } + // skip if mask_path is present but mask is not (segmentations) + if (overlay.label.mask_path && !overlay.label.mask) { + continue; + } + // for heatmaps + if (overlay.label.map_path && !overlay.label.map) { + continue; + } + if (!(overlay.field && overlay.field in bins)) continue; // todo: find a better approach / place for this. From 172b8fb3e5c4720132d870734d35ea2da31e2af5 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Sun, 5 Jan 2025 18:18:25 -0600 Subject: [PATCH 11/37] pass activepaths to worker --- app/packages/looker/src/worker/index.ts | 117 +++++++++++++++++++----- 1 file changed, 95 insertions(+), 22 deletions(-) diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index 71a95d5cf4..907598abdc 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -40,6 +40,14 @@ interface ResolveColor { color: string; } +type DecodeLabelOverlayArgs = { + cls: string; + fieldName: string; + label: Record; + // comes from sample.urls, also known as "sources" elsewhere + urlOverrides?: { [key: string]: string }; +}; + type ResolveColorMethod = ReaderMethod & ResolveColor; const [requestColor, resolveColor] = colorResolve(); @@ -48,8 +56,54 @@ const painterFactory = PainterFactory(requestColor); const ALL_VALID_LABELS = new Set(VALID_LABEL_TYPES); +const decodeLabelOverlay = async ({ + cls, + fieldName, + label, + urlOverrides, +}: DecodeLabelOverlayArgs) => { + const promises = []; + + if (cls === DETECTIONS) { + const promises: Promise[] = []; + for (const detection of label.detections) { + promises.push( + decodeLabelOverlay({ + fieldName, + urlOverrides, + cls: DETECTION, + label: detection, + }) + ); + } + await Promise.all(promises); + return; + } + + const overlayFields = getOverlayFieldFromCls(cls); + const overlayPathField = overlayFields.disk; + const overlayField = overlayFields.canonical; + + if (Boolean(label[overlayField]) || !Object.hasOwn(label, overlayPathField)) { + // it's possible we're just re-coloring, in which case re-init mask image and set bitmap to null + if ( + label[overlayField] && + label[overlayField].bitmap && + !label[overlayField].image + ) { + const height = label[overlayField].bitmap.height; + const width = label[overlayField].bitmap.width; + label[overlayField].image = new ArrayBuffer(height * width * 4); + label[overlayField].bitmap.close(); + label[overlayField].bitmap = null; + } + // nothing to be done + return; + } +}; + /** - * This function processes labels in a recursive manner. It follows the following steps: + * This function processes labels in active paths in a recursive manner. It follows the following steps: * 1. Deserialize masks. Accumulate promises. * 2. Await mask path decoding to finish. * 3. Start painting overlays. Accumulate promises. @@ -67,7 +121,8 @@ const processLabels = async ( colorscale: ProcessSample["colorscale"], labelTagColors: ProcessSample["labelTagColors"], selectedLabelTags: ProcessSample["selectedLabelTags"], - schema: Schema + schema: ProcessSample["schema"], + activePaths: ProcessSample["activePaths"] ): Promise<[Promise[], ArrayBuffer[]]> => { const maskPathDecodingPromises: Promise[] = []; const painterPromises: Promise[] = []; @@ -92,23 +147,32 @@ const processLabels = async ( } if (DENSE_LABELS.has(cls)) { - maskPathDecodingPromises.push( - decodeOverlayOnDisk( - `${prefix || ""}${field}`, - label, - coloring, - customizeColorSetting, - colorscale, - sources, - cls, - maskPathDecodingPromises, - maskTargetsBuffers - ) - ); - } + if ( + activePaths.length && + activePaths.includes(`${prefix ?? ""}${field}`) + ) { + maskPathDecodingPromises.push( + decodeOverlayOnDisk( + `${prefix || ""}${field}`, + label, + coloring, + customizeColorSetting, + colorscale, + sources, + cls, + maskPathDecodingPromises, + maskTargetsBuffers + ) + ); - if (cls in DeserializerFactory) { - DeserializerFactory[cls](label, maskTargetsBuffers); + if (cls in DeserializerFactory) { + DeserializerFactory[cls](label, maskTargetsBuffers); + label.renderStatus = "decoded"; + } + } else { + // we'll process this label asynchronously later + label.renderStatus = null; + } } if ([EMBEDDED_DOCUMENT, DYNAMIC_EMBEDDED_DOCUMENT].includes(cls)) { @@ -122,7 +186,8 @@ const processLabels = async ( colorscale, labelTagColors, selectedLabelTags, - schema + schema, + activePaths ); bitmapPromises.push(...moreBitmapPromises); maskTargetsBuffers.push(...moreMaskTargetsBuffers); @@ -193,7 +258,7 @@ const processLabels = async ( } for (const label of labels) { - if (!label) { + if (label?.renderStatus !== "painted") { continue; } @@ -267,6 +332,7 @@ export interface ProcessSample { selectedLabelTags: string[]; sources: { [path: string]: string }; schema: Schema; + activePaths: string[]; } type ProcessSampleMethod = ReaderMethod & ProcessSample; @@ -281,6 +347,7 @@ const processSample = async ({ selectedLabelTags, labelTagColors, schema, + activePaths, }: ProcessSample) => { mapId(sample); @@ -288,6 +355,7 @@ const processSample = async ({ let maskTargetsBuffers: ArrayBuffer[] = []; if (sample?._media_type === "point-cloud" || sample?._media_type === "3d") { + // we process all 3d labels regardless of active paths process3DLabels(schema, sample); } else { const [bitmapPromises, moreMaskTargetsBuffers] = await processLabels( @@ -299,7 +367,8 @@ const processSample = async ({ colorscale, labelTagColors, selectedLabelTags, - schema + schema, + activePaths ); if (bitmapPromises.length !== 0) { @@ -326,7 +395,8 @@ const processSample = async ({ colorscale, labelTagColors, selectedLabelTags, - schema + schema, + activePaths ) ); } @@ -621,6 +691,9 @@ if (typeof onmessage !== "undefined") { case "processSample": processSample(args as ProcessSample); return; + case "decodeLabelOverlay": + decodeLabelOverlay(args as DecodeLabelOverlayArgs); + return; case "requestFrameChunk": requestFrameChunk(args as RequestFrameChunk); return; From 60eb8fd2f5af912c34a088306a03b2e466502f11 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Sun, 5 Jan 2025 18:18:55 -0600 Subject: [PATCH 12/37] add refreshSample --- app/packages/looker/src/lookers/abstract.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 820a0fe3d8..140508df62 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -23,7 +23,7 @@ import { import { Events } from "../elements/base"; import { COMMON_SHORTCUTS, LookerElement } from "../elements/common"; import { ClassificationsOverlay, loadOverlays } from "../overlays"; -import { CONTAINS, Overlay } from "../overlays/base"; +import { Overlay } from "../overlays/base"; import processOverlays from "../processOverlays"; import { BaseState, @@ -311,9 +311,10 @@ export abstract class AbstractLooker< this.pluckedOverlays ); - this.state.mouseIsOnOverlay = - Boolean(this.currentOverlays.length) && - this.currentOverlays[0].containsPoint(this.state) > CONTAINS.NONE; + // todo: fix me + // this.state.mouseIsOnOverlay = + // Boolean(this.currentOverlays.length) && + // this.currentOverlays[0].containsPoint(this.state) > CONTAINS.NONE; postUpdate?.(this.state, this.currentOverlays, this.sample); @@ -520,6 +521,10 @@ export abstract class AbstractLooker< this.loadSample(sample, retrieveArrayBuffers(this.sampleOverlays)); } + refreshSample() { + this.updateSample(this.sample); + } + getSample(): Promise { const sample = { ...this.sample }; @@ -742,6 +747,7 @@ export abstract class AbstractLooker< sources: this.state.config.sources, schema: this.state.config.fieldSchema, uuid: messageUUID, + activePaths: this.state.options.activePaths, } as ProcessSample; try { From a9390335e960516d3d38e4bf6a9554dd4ccf210f Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Sun, 5 Jan 2025 19:16:11 -0600 Subject: [PATCH 13/37] add render status --- .../looker/src/worker/disk-overlay-decoder.ts | 13 +++++-------- app/packages/looker/src/worker/painter.ts | 16 ++++++++++++++++ app/packages/looker/src/worker/shared.ts | 2 ++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/app/packages/looker/src/worker/disk-overlay-decoder.ts b/app/packages/looker/src/worker/disk-overlay-decoder.ts index 2a8ab04b66..ad61cb3aa2 100644 --- a/app/packages/looker/src/worker/disk-overlay-decoder.ts +++ b/app/packages/looker/src/worker/disk-overlay-decoder.ts @@ -1,16 +1,11 @@ import { getSampleSrc } from "@fiftyone/state/src/recoil/utils"; -import { - DETECTION, - DETECTIONS, - HEATMAP, - SEGMENTATION, -} from "@fiftyone/utilities"; +import { DETECTION, DETECTIONS } from "@fiftyone/utilities"; import { Coloring, CustomizeColor } from ".."; import { OverlayMask } from "../numpy"; import { Colorscale } from "../state"; -import { decodeWithCanvas, recastBufferToMonoChannel } from "./canvas-decoder"; +import { decodeWithCanvas } from "./canvas-decoder"; import { enqueueFetch } from "./pooled-fetch"; -import { getOverlayFieldFromCls } from "./shared"; +import { DenseLabelRenderStatus, getOverlayFieldFromCls } from "./shared"; export type IntermediateMask = { data: OverlayMask; @@ -136,6 +131,8 @@ export const decodeOverlayOnDisk = async ( image: new ArrayBuffer(overlayWidth * overlayHeight * 4), } as IntermediateMask; + label.renderStatus = "decoded" as DenseLabelRenderStatus; + // no need to transfer image's buffer //since we'll be constructing ImageBitmap and transfering that maskTargetsBuffers.push(overlayMask.buffer); diff --git a/app/packages/looker/src/worker/painter.ts b/app/packages/looker/src/worker/painter.ts index 1e8675aa5a..6a578396b0 100644 --- a/app/packages/looker/src/worker/painter.ts +++ b/app/packages/looker/src/worker/painter.ts @@ -30,6 +30,8 @@ export const PainterFactory = (requestColor) => ({ return; } + label.renderStatus = "painting"; + const setting = customizeColorSetting?.find((s) => s.path === field); let color; @@ -141,6 +143,8 @@ export const PainterFactory = (requestColor) => ({ } } } + + label.renderStatus = "painted"; }, Detections: async ( field, @@ -155,6 +159,8 @@ export const PainterFactory = (requestColor) => ({ return; } + labels.renderStatus = "painting"; + const promises = labels.detections.map((label) => PainterFactory(requestColor).Detection( field, @@ -168,6 +174,8 @@ export const PainterFactory = (requestColor) => ({ ); await Promise.all(promises); + + labels.renderStatus = "painted"; }, Heatmap: async ( field, @@ -182,6 +190,8 @@ export const PainterFactory = (requestColor) => ({ return; } + label.renderStatus = "painting"; + const overlay = new Uint32Array(label.map.image); const mapData = label.map.data; @@ -245,6 +255,8 @@ export const PainterFactory = (requestColor) => ({ overlay[i] = r; } + + label.renderStatus = "painted"; }, Segmentation: async ( field, @@ -259,6 +271,8 @@ export const PainterFactory = (requestColor) => ({ return; } + label.renderStatus = "painting"; + // the actual overlay that'll be painted, byte-length of width * height * 4 (RGBA channels) const overlay = new Uint32Array(label.mask.image); @@ -390,6 +404,8 @@ export const PainterFactory = (requestColor) => ({ } } } + + label.renderStatus = "painted"; }, }); diff --git a/app/packages/looker/src/worker/shared.ts b/app/packages/looker/src/worker/shared.ts index ec383b7536..906014053b 100644 --- a/app/packages/looker/src/worker/shared.ts +++ b/app/packages/looker/src/worker/shared.ts @@ -1,5 +1,7 @@ import { HEATMAP } from "@fiftyone/utilities"; +export type DenseLabelRenderStatus = "decoded" | "decoding" | "undecoded"; + /** * Map the _id field to id */ From 2aa368b4ffa4a5c090cf2965c3f093728c526c54 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Sun, 5 Jan 2025 20:09:51 -0600 Subject: [PATCH 14/37] remove redundant guard --- app/packages/looker/src/processOverlays.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/packages/looker/src/processOverlays.ts b/app/packages/looker/src/processOverlays.ts index a99dc5939a..f02ffd6db4 100644 --- a/app/packages/looker/src/processOverlays.ts +++ b/app/packages/looker/src/processOverlays.ts @@ -30,15 +30,6 @@ const processOverlays = ( continue; } - // skip if mask_path is present but mask is not (segmentations) - if (overlay.label.mask_path && !overlay.label.mask) { - continue; - } - // for heatmaps - if (overlay.label.map_path && !overlay.label.map) { - continue; - } - if (!(overlay.field && overlay.field in bins)) continue; // todo: find a better approach / place for this. From dd869ccef47534a8090b37abcb8d3a60ceeb9e8f Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Sun, 5 Jan 2025 20:22:39 -0600 Subject: [PATCH 15/37] refresh layout when sidebar is updated --- .../core/src/components/Grid/Grid.tsx | 3 + .../core/src/components/Grid/useRefreshers.ts | 35 +++++++---- .../Sidebar/useOnSidebarSelectionChange.ts | 63 +++++++++++++++++++ app/packages/state/src/recoil/sidebar.ts | 9 +++ 4 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts diff --git a/app/packages/core/src/components/Grid/Grid.tsx b/app/packages/core/src/components/Grid/Grid.tsx index 8948d8b1ff..c45cfb8507 100644 --- a/app/packages/core/src/components/Grid/Grid.tsx +++ b/app/packages/core/src/components/Grid/Grid.tsx @@ -13,6 +13,7 @@ import React, { import { useRecoilValue } from "recoil"; import { v4 as uuid } from "uuid"; import { QP_WAIT, QueryPerformanceToastEvent } from "../QueryPerformanceToast"; +import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange"; import { gridCrop, gridSpacing, pageParameters } from "./recoil"; import useAt from "./useAt"; import useEscape from "./useEscape"; @@ -46,6 +47,8 @@ function Grid() { const setSample = fos.useExpandSample(store); const getFontSize = useFontSize(id); + useOnSidebarSelectionChange(); + const spotlight = useMemo(() => { /** SPOTLIGHT REFRESHER */ reset; diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index 1fa1b38d09..d980092dc3 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -29,14 +29,23 @@ export default function useRefreshers() { ); const view = fos.filterView(useRecoilValue(fos.view)); + const labelsToggleTracker = useRecoilValue(fos.labelsToggleTracker); + // only reload, attempt to return to the last grid location const layoutReset = useMemo(() => { cropToContent; fieldVisibilityStage; + labelsToggleTracker; mediaField; refresher; return uuid(); - }, [cropToContent, fieldVisibilityStage, mediaField, refresher]); + }, [ + cropToContent, + fieldVisibilityStage, + labelsToggleTracker, + mediaField, + refresher, + ]); // the values reset the page, i.e. return to the top const pageReset = useMemo(() => { @@ -64,18 +73,20 @@ export default function useRefreshers() { return uuid(); }, [layoutReset, pageReset]); - useEffect( - () => - subscribe(({ event }, { reset }) => { - if (event === "fieldVisibility") return; + useEffect(() => { + const unsubscribe = subscribe(({ event }, { reset }) => { + if (event === "fieldVisibility") return; - // if not a modal page change, reset the grid location - reset(gridAt); - reset(gridPage); - reset(gridOffset); - }), - [] - ); + // if not a modal page change, reset the grid location + reset(gridAt); + reset(gridPage); + reset(gridOffset); + }); + + return () => { + unsubscribe(); + }; + }, []); const lookerStore = useMemo(() => { /** LOOKER STORE REFRESHER */ diff --git a/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts new file mode 100644 index 0000000000..f74d773e86 --- /dev/null +++ b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts @@ -0,0 +1,63 @@ +import { activeLabelFields, labelsToggleTracker } from "@fiftyone/state"; +import { useEffect, useRef } from "react"; +import { useRecoilValue, useSetRecoilState } from "recoil"; +import { gridPage } from "../Grid/recoil"; + +/** + * This hook is used to update the sidebar tracker when the user changes the + * selection of labels in the sidebar. We keep a map of grid page to the active + * label fields for that page. + */ +export const useOnSidebarSelectionChange = () => { + const activeLabelFieldsValue = useRecoilValue( + activeLabelFields({ modal: false }) + ); + const gridPageValue = useRecoilValue(gridPage); + + const gridPageValueRef = useRef(gridPageValue); + + gridPageValueRef.current = gridPageValue; + + const setSidebarTracker = useSetRecoilState(labelsToggleTracker); + + const debugSidebarTracker = useRecoilValue(labelsToggleTracker); + + useEffect(() => { + const thisPageActiveFields = debugSidebarTracker.get(gridPageValue); + + const currentActiveLabelFields = new Set(activeLabelFieldsValue); + + if (currentActiveLabelFields.size === 0) { + return; + } + + // diff the two sets, we only care about net new fields + // if there are no new fields, we don't need to update the tracker + let hasNewFields = false; + if (thisPageActiveFields) { + for (const field of currentActiveLabelFields) { + if (!thisPageActiveFields.has(field)) { + hasNewFields = true; + break; + } + } + } else { + hasNewFields = true; + } + + if (!hasNewFields) { + return; + } + + const newTracker = new Map([ + ...debugSidebarTracker, + [gridPageValueRef.current, new Set(activeLabelFieldsValue)], + ]); + + if (newTracker.size === 0) { + return; + } + + setSidebarTracker(newTracker); + }, [activeLabelFieldsValue, debugSidebarTracker]); +}; diff --git a/app/packages/state/src/recoil/sidebar.ts b/app/packages/state/src/recoil/sidebar.ts index d92783dbdd..e3d8f4d002 100644 --- a/app/packages/state/src/recoil/sidebar.ts +++ b/app/packages/state/src/recoil/sidebar.ts @@ -29,6 +29,7 @@ import type { VariablesOf } from "react-relay"; import { commitMutation } from "react-relay"; import { DefaultValue, + atom, atomFamily, selector, selectorFamily, @@ -72,6 +73,9 @@ import { } from "./utils"; import * as viewAtoms from "./view"; +type GridPageNumber = number; +type SidebarEntriesSet = Set; + export enum EntryKind { EMPTY = "EMPTY", GROUP = "GROUP", @@ -906,6 +910,11 @@ export const groupShown = selectorFamily< }, }); +export const labelsToggleTracker = atom({ + key: "labelsToggleTracker", + default: new Map(), +}); + export const textFilter = atomFamily({ key: "textFilter", default: "", From 2c408042d65220694b8d2290c2b30e2b576592e2 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Sun, 5 Jan 2025 20:24:58 -0600 Subject: [PATCH 16/37] remove decodeLabelOverlay --- .../core/src/components/Grid/useRefreshers.ts | 16 +++--- .../core/src/components/Grid/useSelect.ts | 1 + app/packages/looker/src/worker/index.ts | 57 ------------------- 3 files changed, 9 insertions(+), 65 deletions(-) diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index d980092dc3..d6cf60a82c 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -35,17 +35,10 @@ export default function useRefreshers() { const layoutReset = useMemo(() => { cropToContent; fieldVisibilityStage; - labelsToggleTracker; mediaField; refresher; return uuid(); - }, [ - cropToContent, - fieldVisibilityStage, - labelsToggleTracker, - mediaField, - refresher, - ]); + }, [cropToContent, fieldVisibilityStage, mediaField, refresher]); // the values reset the page, i.e. return to the top const pageReset = useMemo(() => { @@ -102,6 +95,13 @@ export default function useRefreshers() { }); }, [reset]); + useEffect(() => { + console.log(">>>sashank") + lookerStore.forEach((looker) => { + looker.refreshSample(); + }); + }, [labelsToggleTracker]); + return { lookerStore, pageReset, diff --git a/app/packages/core/src/components/Grid/useSelect.ts b/app/packages/core/src/components/Grid/useSelect.ts index 7958c8b219..c530bc635f 100644 --- a/app/packages/core/src/components/Grid/useSelect.ts +++ b/app/packages/core/src/components/Grid/useSelect.ts @@ -14,6 +14,7 @@ export default function useSelect( const selected = useRecoilValue(fos.selectedSamples); useEffect(() => { + console.log(">>>ben"); deferred(() => { const fontSize = getFontSize(); const retained = new Set(); diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index 907598abdc..f255d8ca73 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -40,14 +40,6 @@ interface ResolveColor { color: string; } -type DecodeLabelOverlayArgs = { - cls: string; - fieldName: string; - label: Record; - // comes from sample.urls, also known as "sources" elsewhere - urlOverrides?: { [key: string]: string }; -}; - type ResolveColorMethod = ReaderMethod & ResolveColor; const [requestColor, resolveColor] = colorResolve(); @@ -56,52 +48,6 @@ const painterFactory = PainterFactory(requestColor); const ALL_VALID_LABELS = new Set(VALID_LABEL_TYPES); -const decodeLabelOverlay = async ({ - cls, - fieldName, - label, - urlOverrides, -}: DecodeLabelOverlayArgs) => { - const promises = []; - - if (cls === DETECTIONS) { - const promises: Promise[] = []; - for (const detection of label.detections) { - promises.push( - decodeLabelOverlay({ - fieldName, - urlOverrides, - cls: DETECTION, - label: detection, - }) - ); - } - await Promise.all(promises); - return; - } - - const overlayFields = getOverlayFieldFromCls(cls); - const overlayPathField = overlayFields.disk; - const overlayField = overlayFields.canonical; - - if (Boolean(label[overlayField]) || !Object.hasOwn(label, overlayPathField)) { - // it's possible we're just re-coloring, in which case re-init mask image and set bitmap to null - if ( - label[overlayField] && - label[overlayField].bitmap && - !label[overlayField].image - ) { - const height = label[overlayField].bitmap.height; - const width = label[overlayField].bitmap.width; - label[overlayField].image = new ArrayBuffer(height * width * 4); - label[overlayField].bitmap.close(); - label[overlayField].bitmap = null; - } - // nothing to be done - return; - } -}; - /** * This function processes labels in active paths in a recursive manner. It follows the following steps: * 1. Deserialize masks. Accumulate promises. @@ -691,9 +637,6 @@ if (typeof onmessage !== "undefined") { case "processSample": processSample(args as ProcessSample); return; - case "decodeLabelOverlay": - decodeLabelOverlay(args as DecodeLabelOverlayArgs); - return; case "requestFrameChunk": requestFrameChunk(args as RequestFrameChunk); return; From 90d80663092171610f154b8759161d112a9f01da Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 16:45:39 -0600 Subject: [PATCH 17/37] fix modal --- app/packages/core/src/components/Grid/Grid.tsx | 3 --- .../core/src/components/Grid/useRefreshers.ts | 18 ++++++++++++------ .../core/src/components/Modal/Modal.tsx | 9 ++++++++- .../Sidebar/useOnSidebarSelectionChange.ts | 18 +++++++++--------- app/packages/state/src/recoil/sidebar.ts | 6 ++++-- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/app/packages/core/src/components/Grid/Grid.tsx b/app/packages/core/src/components/Grid/Grid.tsx index c45cfb8507..8948d8b1ff 100644 --- a/app/packages/core/src/components/Grid/Grid.tsx +++ b/app/packages/core/src/components/Grid/Grid.tsx @@ -13,7 +13,6 @@ import React, { import { useRecoilValue } from "recoil"; import { v4 as uuid } from "uuid"; import { QP_WAIT, QueryPerformanceToastEvent } from "../QueryPerformanceToast"; -import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange"; import { gridCrop, gridSpacing, pageParameters } from "./recoil"; import useAt from "./useAt"; import useEscape from "./useEscape"; @@ -47,8 +46,6 @@ function Grid() { const setSample = fos.useExpandSample(store); const getFontSize = useFontSize(id); - useOnSidebarSelectionChange(); - const spotlight = useMemo(() => { /** SPOTLIGHT REFRESHER */ reset; diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index d6cf60a82c..c1cb65f362 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -1,9 +1,10 @@ import { subscribe } from "@fiftyone/relay"; import * as fos from "@fiftyone/state"; import { LRUCache } from "lru-cache"; -import { useEffect, useMemo } from "react"; +import { useEffect, useMemo, useRef } from "react"; import uuid from "react-uuid"; import { useRecoilValue } from "recoil"; +import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange"; import { gridAt, gridOffset, gridPage } from "./recoil"; const MAX_LRU_CACHE_ITEMS = 510; @@ -29,8 +30,6 @@ export default function useRefreshers() { ); const view = fos.filterView(useRecoilValue(fos.view)); - const labelsToggleTracker = useRecoilValue(fos.labelsToggleTracker); - // only reload, attempt to return to the last grid location const layoutReset = useMemo(() => { cropToContent; @@ -95,12 +94,19 @@ export default function useRefreshers() { }); }, [reset]); + const lookerStoreRef = useRef(lookerStore); + lookerStoreRef.current = lookerStore; + + const gridLabelsToggleTracker = useOnSidebarSelectionChange({ modal: false }); + + /** + * Refresh all lookers when the labels toggle tracker changes + */ useEffect(() => { - console.log(">>>sashank") - lookerStore.forEach((looker) => { + lookerStoreRef.current?.forEach((looker) => { looker.refreshSample(); }); - }, [labelsToggleTracker]); + }, [gridLabelsToggleTracker]); return { lookerStore, diff --git a/app/packages/core/src/components/Modal/Modal.tsx b/app/packages/core/src/components/Modal/Modal.tsx index 6443fa8a2b..426d37e54e 100644 --- a/app/packages/core/src/components/Modal/Modal.tsx +++ b/app/packages/core/src/components/Modal/Modal.tsx @@ -1,12 +1,13 @@ import { HelpPanel, JSONPanel } from "@fiftyone/components"; import { OPERATOR_PROMPT_AREAS, OperatorPromptArea } from "@fiftyone/operators"; import * as fos from "@fiftyone/state"; -import React, { useCallback, useMemo, useRef } from "react"; +import React, { useCallback, useEffect, useMemo, useRef } from "react"; import ReactDOM from "react-dom"; import { useRecoilCallback, useRecoilValue } from "recoil"; import styled from "styled-components"; import { ModalActionsRow } from "../Actions"; import Sidebar from "../Sidebar"; +import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange"; import ModalNavigation from "./ModalNavigation"; import { ModalSpace } from "./ModalSpace"; import { TooltipInfo } from "./TooltipInfo"; @@ -178,6 +179,12 @@ const Modal = () => { const activeLookerRef = useRef(); + const labelsToggleTracker = useOnSidebarSelectionChange({ modal: true }); + + useEffect(() => { + activeLookerRef.current?.refreshSample(); + }, [labelsToggleTracker]); + const addTooltipEventHandler = useTooltipEventHandler(); const removeTooltipEventHanlderRef = useRef { - const activeLabelFieldsValue = useRecoilValue( - activeLabelFields({ modal: false }) - ); +export const useOnSidebarSelectionChange = ({ modal }: { modal: boolean }) => { + const activeLabelFieldsValue = useRecoilValue(activeLabelFields({ modal })); const gridPageValue = useRecoilValue(gridPage); const gridPageValueRef = useRef(gridPageValue); gridPageValueRef.current = gridPageValue; - const setSidebarTracker = useSetRecoilState(labelsToggleTracker); + const setSidebarTracker = useSetRecoilState(labelsToggleTracker(modal)); - const debugSidebarTracker = useRecoilValue(labelsToggleTracker); + const sidebarTracker = useRecoilValue(labelsToggleTracker(modal)); useEffect(() => { - const thisPageActiveFields = debugSidebarTracker.get(gridPageValue); + const thisPageActiveFields = sidebarTracker.get(gridPageValue); const currentActiveLabelFields = new Set(activeLabelFieldsValue); @@ -50,7 +48,7 @@ export const useOnSidebarSelectionChange = () => { } const newTracker = new Map([ - ...debugSidebarTracker, + ...sidebarTracker, [gridPageValueRef.current, new Set(activeLabelFieldsValue)], ]); @@ -59,5 +57,7 @@ export const useOnSidebarSelectionChange = () => { } setSidebarTracker(newTracker); - }, [activeLabelFieldsValue, debugSidebarTracker]); + }, [activeLabelFieldsValue, sidebarTracker]); + + return sidebarTracker; }; diff --git a/app/packages/state/src/recoil/sidebar.ts b/app/packages/state/src/recoil/sidebar.ts index e3d8f4d002..89525a08e9 100644 --- a/app/packages/state/src/recoil/sidebar.ts +++ b/app/packages/state/src/recoil/sidebar.ts @@ -29,7 +29,6 @@ import type { VariablesOf } from "react-relay"; import { commitMutation } from "react-relay"; import { DefaultValue, - atom, atomFamily, selector, selectorFamily, @@ -910,7 +909,10 @@ export const groupShown = selectorFamily< }, }); -export const labelsToggleTracker = atom({ +export const labelsToggleTracker = atomFamily< + Map, + boolean +>({ key: "labelsToggleTracker", default: new Map(), }); From a3f376bdd43d38e875da6f7cadba68e68ec50e78 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 16:52:02 -0600 Subject: [PATCH 18/37] better name --- app/packages/core/src/components/Modal/Modal.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/packages/core/src/components/Modal/Modal.tsx b/app/packages/core/src/components/Modal/Modal.tsx index 426d37e54e..bdfc8cbcb5 100644 --- a/app/packages/core/src/components/Modal/Modal.tsx +++ b/app/packages/core/src/components/Modal/Modal.tsx @@ -179,11 +179,11 @@ const Modal = () => { const activeLookerRef = useRef(); - const labelsToggleTracker = useOnSidebarSelectionChange({ modal: true }); + const modalLabelsToggleTracker = useOnSidebarSelectionChange({ modal: true }); useEffect(() => { activeLookerRef.current?.refreshSample(); - }, [labelsToggleTracker]); + }, [modalLabelsToggleTracker]); const addTooltipEventHandler = useTooltipEventHandler(); const removeTooltipEventHanlderRef = useRef Date: Tue, 7 Jan 2025 16:56:43 -0600 Subject: [PATCH 19/37] Remove unnecessary log --- app/packages/core/src/components/Grid/useSelect.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/packages/core/src/components/Grid/useSelect.ts b/app/packages/core/src/components/Grid/useSelect.ts index c530bc635f..7958c8b219 100644 --- a/app/packages/core/src/components/Grid/useSelect.ts +++ b/app/packages/core/src/components/Grid/useSelect.ts @@ -14,7 +14,6 @@ export default function useSelect( const selected = useRecoilValue(fos.selectedSamples); useEffect(() => { - console.log(">>>ben"); deferred(() => { const fontSize = getFontSize(); const retained = new Set(); From 419b02bfcd352161b16bdbad8fa6dbb07ec11847 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 17:39:16 -0600 Subject: [PATCH 20/37] retry update sample --- app/packages/looker/src/lookers/abstract.ts | 16 ++++++++++++++-- app/packages/looker/src/overlays/base.ts | 2 +- app/packages/looker/src/overlays/detection.ts | 6 +++++- app/packages/looker/src/overlays/heatmap.ts | 7 +++++-- app/packages/looker/src/overlays/segmentation.ts | 7 +++++-- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 140508df62..2169b1963c 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -46,6 +46,8 @@ import { ProcessSample } from "../worker"; import { LookerUtils } from "./shared"; import { retrieveArrayBuffers } from "./utils"; +const UPDATING_SAMPLES_IDS = new Set(); + const LABEL_LISTS_PATH = new Set(withPath(LABELS_PATH, LABEL_LISTS)); const LABEL_LIST_KEY = Object.fromEntries( Object.entries(LABEL_LISTS_MAP).map(([k, v]) => [withPath(LABELS_PATH, k), v]) @@ -518,6 +520,14 @@ export abstract class AbstractLooker< abstract updateOptions(options: Partial): void; updateSample(sample: Sample) { + if (UPDATING_SAMPLES_IDS.has(sample.id)) { + UPDATING_SAMPLES_IDS.delete(sample.id); + this.cleanOverlays(true); + queueMicrotask(() => this.updateSample(sample)); + return; + } + + UPDATING_SAMPLES_IDS.add(sample.id); this.loadSample(sample, retrieveArrayBuffers(this.sampleOverlays)); } @@ -706,9 +716,9 @@ export abstract class AbstractLooker< ); } - protected cleanOverlays() { + protected cleanOverlays(setTargetsToNull = false) { for (const overlay of this.sampleOverlays ?? []) { - overlay.cleanup?.(); + overlay.cleanup?.(setTargetsToNull); } } @@ -731,6 +741,8 @@ export abstract class AbstractLooker< reloading: false, }); labelsWorker.removeEventListener("message", listener); + + UPDATING_SAMPLES_IDS.delete(sample.id); } }; diff --git a/app/packages/looker/src/overlays/base.ts b/app/packages/looker/src/overlays/base.ts index 94a7eb6683..bfc34c69f9 100644 --- a/app/packages/looker/src/overlays/base.ts +++ b/app/packages/looker/src/overlays/base.ts @@ -74,7 +74,7 @@ export interface Overlay> { getPoints(state: Readonly): Coordinates[]; getSelectData(state: Readonly): SelectData; getSizeBytes(): number; - cleanup?(): void; + cleanup?(setTargetsToNull: boolean): void; } export abstract class CoordinateOverlay< diff --git a/app/packages/looker/src/overlays/detection.ts b/app/packages/looker/src/overlays/detection.ts index f17535a342..d5abe24bfc 100644 --- a/app/packages/looker/src/overlays/detection.ts +++ b/app/packages/looker/src/overlays/detection.ts @@ -261,8 +261,12 @@ export default class DetectionOverlay< return [(bx - ow) * w, (by - oh) * h, (bw + ow * 2) * w, (bh + oh * 2) * h]; } - public cleanup(): void { + public cleanup(setTargetsToNull = false): void { this.label.mask?.bitmap?.close(); + + if (setTargetsToNull) { + this.label.mask = null; + } } } diff --git a/app/packages/looker/src/overlays/heatmap.ts b/app/packages/looker/src/overlays/heatmap.ts index b2a7a67e96..5261816446 100644 --- a/app/packages/looker/src/overlays/heatmap.ts +++ b/app/packages/looker/src/overlays/heatmap.ts @@ -206,9 +206,12 @@ export default class HeatmapOverlay return sizeBytesEstimate(this.label); } - public cleanup(): void { + public cleanup(setTargetsToNull = false): void { this.label.map?.bitmap?.close(); - this.targets = null; + + if (setTargetsToNull) { + this.targets = null; + } } } diff --git a/app/packages/looker/src/overlays/segmentation.ts b/app/packages/looker/src/overlays/segmentation.ts index e06d091cbc..73df49b1d3 100644 --- a/app/packages/looker/src/overlays/segmentation.ts +++ b/app/packages/looker/src/overlays/segmentation.ts @@ -261,9 +261,12 @@ export default class SegmentationOverlay return sizeBytesEstimate(this.label); } - public cleanup(): void { + public cleanup(setTargetsToNull = false): void { this.label.mask?.bitmap?.close(); - this.targets = null; + + if (setTargetsToNull) { + this.label.mask = null; + } } } From 5e85c2b9bfe333329f5ed3014342b9ccc69a5317 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 17:42:26 -0600 Subject: [PATCH 21/37] add painting status --- app/packages/looker/src/worker/shared.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/looker/src/worker/shared.ts b/app/packages/looker/src/worker/shared.ts index 906014053b..618be3fe47 100644 --- a/app/packages/looker/src/worker/shared.ts +++ b/app/packages/looker/src/worker/shared.ts @@ -1,6 +1,6 @@ import { HEATMAP } from "@fiftyone/utilities"; -export type DenseLabelRenderStatus = "decoded" | "decoding" | "undecoded"; +export type DenseLabelRenderStatus = null | "painting" | "painted" | "decoded"; /** * Map the _id field to id From 3c8704245bfa92c53d4e15763ed99234f6a4c08a Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 18:20:49 -0600 Subject: [PATCH 22/37] add guard in painting loop --- app/packages/looker/src/worker/index.ts | 30 +++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index f255d8ca73..4817130f0a 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -171,18 +171,24 @@ const processLabels = async ( if (!label) { continue; } - if (painterFactory[cls]) { - painterPromises.push( - painterFactory[cls]( - prefix ? prefix + field : field, - label, - coloring, - customizeColorSetting, - colorscale, - labelTagColors, - selectedLabelTags - ) - ); + + if ( + activePaths.length && + activePaths.includes(`${prefix ?? ""}${field}`) + ) { + if (painterFactory[cls]) { + painterPromises.push( + painterFactory[cls]( + prefix ? prefix + field : field, + label, + coloring, + customizeColorSetting, + colorscale, + labelTagColors, + selectedLabelTags + ) + ); + } } } } From c88993fa8c4b508be7c547bd9e3728f4a4a2a02a Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 18:27:27 -0600 Subject: [PATCH 23/37] use local recoil state --- .../Sidebar/useOnSidebarSelectionChange.ts | 15 ++++++++++----- app/packages/state/src/recoil/sidebar.ts | 11 ----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts index 6a81d74dc8..bea5539c33 100644 --- a/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts +++ b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts @@ -1,8 +1,13 @@ -import { activeLabelFields, labelsToggleTracker } from "@fiftyone/state"; +import { activeLabelFields } from "@fiftyone/state"; import { useEffect, useRef } from "react"; -import { useRecoilValue, useSetRecoilState } from "recoil"; +import { atomFamily, useRecoilState, useRecoilValue } from "recoil"; import { gridPage } from "../Grid/recoil"; +const labelsToggleTracker = atomFamily>, boolean>({ + key: "labelsToggleTracker", + default: new Map>(), +}); + /** * This hook is used to update the sidebar tracker when the user changes the * selection of labels in the sidebar. We keep a map of grid page to the active @@ -16,9 +21,9 @@ export const useOnSidebarSelectionChange = ({ modal }: { modal: boolean }) => { gridPageValueRef.current = gridPageValue; - const setSidebarTracker = useSetRecoilState(labelsToggleTracker(modal)); - - const sidebarTracker = useRecoilValue(labelsToggleTracker(modal)); + const [sidebarTracker, setSidebarTracker] = useRecoilState( + labelsToggleTracker(modal) + ); useEffect(() => { const thisPageActiveFields = sidebarTracker.get(gridPageValue); diff --git a/app/packages/state/src/recoil/sidebar.ts b/app/packages/state/src/recoil/sidebar.ts index 89525a08e9..d92783dbdd 100644 --- a/app/packages/state/src/recoil/sidebar.ts +++ b/app/packages/state/src/recoil/sidebar.ts @@ -72,9 +72,6 @@ import { } from "./utils"; import * as viewAtoms from "./view"; -type GridPageNumber = number; -type SidebarEntriesSet = Set; - export enum EntryKind { EMPTY = "EMPTY", GROUP = "GROUP", @@ -909,14 +906,6 @@ export const groupShown = selectorFamily< }, }); -export const labelsToggleTracker = atomFamily< - Map, - boolean ->({ - key: "labelsToggleTracker", - default: new Map(), -}); - export const textFilter = atomFamily({ key: "textFilter", default: "", From 2b13086605b4e14908e2963fb2d94ac6ea8c2147 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 18:29:28 -0600 Subject: [PATCH 24/37] remove fix me guard --- app/packages/looker/src/lookers/abstract.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 2169b1963c..94b23e208d 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -23,7 +23,7 @@ import { import { Events } from "../elements/base"; import { COMMON_SHORTCUTS, LookerElement } from "../elements/common"; import { ClassificationsOverlay, loadOverlays } from "../overlays"; -import { Overlay } from "../overlays/base"; +import { CONTAINS, Overlay } from "../overlays/base"; import processOverlays from "../processOverlays"; import { BaseState, @@ -313,10 +313,9 @@ export abstract class AbstractLooker< this.pluckedOverlays ); - // todo: fix me - // this.state.mouseIsOnOverlay = - // Boolean(this.currentOverlays.length) && - // this.currentOverlays[0].containsPoint(this.state) > CONTAINS.NONE; + this.state.mouseIsOnOverlay = + Boolean(this.currentOverlays.length) && + this.currentOverlays[0].containsPoint(this.state) > CONTAINS.NONE; postUpdate?.(this.state, this.currentOverlays, this.sample); From 7b80b337f18f705873b381f3534b660d7317a94c Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Tue, 7 Jan 2025 18:35:18 -0600 Subject: [PATCH 25/37] use lru cache in color resolver --- app/packages/looker/src/worker/resolve-color.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/packages/looker/src/worker/resolve-color.ts b/app/packages/looker/src/worker/resolve-color.ts index aabb02c60b..0e217ecbb8 100644 --- a/app/packages/looker/src/worker/resolve-color.ts +++ b/app/packages/looker/src/worker/resolve-color.ts @@ -1,3 +1,7 @@ +import { LRUCache } from "lru-cache"; + +const MAX_COLOR_CACHE_SIZE = 1000; + export interface ResolveColor { key: string | number; seed: number; @@ -8,13 +12,16 @@ export default (): [ (pool: string[], seed: number, key: string | number) => Promise, (result: ResolveColor) => void ] => { - const cache = {}; + const cache = new LRUCache({ + max: MAX_COLOR_CACHE_SIZE, + }); + const requests = {}; const promises = {}; return [ (pool, seed, key) => { - if (!(seed in cache)) { + if (!cache.has(seed)) { cache[seed] = {}; } From 9cd399a6cf4fc13425ebcf817f4fae158339cb56 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 12:26:29 -0600 Subject: [PATCH 26/37] add null check in visibility labels selector --- app/packages/state/src/recoil/selectors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/state/src/recoil/selectors.ts b/app/packages/state/src/recoil/selectors.ts index ddd239d934..626656be0a 100644 --- a/app/packages/state/src/recoil/selectors.ts +++ b/app/packages/state/src/recoil/selectors.ts @@ -147,7 +147,7 @@ export const defaultVisibilityLabels = key: "defaultVisibilityLabels", get: ({ get }) => { return get(datasetAppConfig) - .defaultVisibilityLabels as State.DefaultVisibilityLabelsConfig | null; + ?.defaultVisibilityLabels as State.DefaultVisibilityLabelsConfig | null; }, }); From c530a707c17840be50da91dd5ce044508c0c4728 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 12:32:47 -0600 Subject: [PATCH 27/37] sample id fallback to _id --- app/packages/looker/src/lookers/abstract.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 94b23e208d..7ba2bc35cb 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -519,14 +519,15 @@ export abstract class AbstractLooker< abstract updateOptions(options: Partial): void; updateSample(sample: Sample) { - if (UPDATING_SAMPLES_IDS.has(sample.id)) { - UPDATING_SAMPLES_IDS.delete(sample.id); + const id = sample.id ?? sample._id; + if (UPDATING_SAMPLES_IDS.has(id)) { + UPDATING_SAMPLES_IDS.delete(id); this.cleanOverlays(true); queueMicrotask(() => this.updateSample(sample)); return; } - UPDATING_SAMPLES_IDS.add(sample.id); + UPDATING_SAMPLES_IDS.add(id); this.loadSample(sample, retrieveArrayBuffers(this.sampleOverlays)); } @@ -741,7 +742,7 @@ export abstract class AbstractLooker< }); labelsWorker.removeEventListener("message", listener); - UPDATING_SAMPLES_IDS.delete(sample.id); + UPDATING_SAMPLES_IDS.delete(sample.id ?? sample._id); } }; From b41bc3f1f3f2740ecd9cb37025b13494bbf5b9fc Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 12:33:08 -0600 Subject: [PATCH 28/37] sample id fallback to _id --- app/packages/looker/src/lookers/abstract.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 7ba2bc35cb..093c270c35 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -520,6 +520,7 @@ export abstract class AbstractLooker< updateSample(sample: Sample) { const id = sample.id ?? sample._id; + if (UPDATING_SAMPLES_IDS.has(id)) { UPDATING_SAMPLES_IDS.delete(id); this.cleanOverlays(true); @@ -528,6 +529,7 @@ export abstract class AbstractLooker< } UPDATING_SAMPLES_IDS.add(id); + this.loadSample(sample, retrieveArrayBuffers(this.sampleOverlays)); } From 65a553bafd079bb5caf69c313501329caacbb6bb Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 14:32:58 -0600 Subject: [PATCH 29/37] pass activepaths in frames --- app/packages/looker/src/lookers/frame-reader.ts | 3 +++ app/packages/looker/src/lookers/video.ts | 1 + app/packages/looker/src/worker/index.ts | 10 +++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/packages/looker/src/lookers/frame-reader.ts b/app/packages/looker/src/lookers/frame-reader.ts index ce0489dc43..db7e81fe3a 100644 --- a/app/packages/looker/src/lookers/frame-reader.ts +++ b/app/packages/looker/src/lookers/frame-reader.ts @@ -32,6 +32,7 @@ export interface Frame { } interface AcquireReaderOptions { + activePaths: string[]; addFrame: (frameNumber: number, frame: Frame) => void; addFrameBuffers: (range: [number, number]) => void; coloring: Coloring; @@ -88,6 +89,7 @@ export const { acquireReader, clearReader } = (() => { view, group, schema, + activePaths, }: AcquireReaderOptions): string => { nextRange = [frameNumber, Math.min(frameCount, CHUNK_SIZE + frameNumber)]; const subscription = uuid(); @@ -130,6 +132,7 @@ export const { acquireReader, clearReader } = (() => { requestingFrames = true; frameReader.postMessage({ method: "setStream", + activePaths, sampleId, frameCount, frameNumber, diff --git a/app/packages/looker/src/lookers/video.ts b/app/packages/looker/src/lookers/video.ts index d1f94328b4..51172f9d82 100644 --- a/app/packages/looker/src/lookers/video.ts +++ b/app/packages/looker/src/lookers/video.ts @@ -257,6 +257,7 @@ export class VideoLooker extends AbstractLooker { addFrameBuffers: (range) => { this.state.buffers = addToBuffers(range, this.state.buffers); }, + activePaths: this.state.options.activePaths, coloring: this.state.options.coloring, customizeColorSetting: this.state.options.customizeColorSetting, dispatchEvent: (event, detail) => this.dispatchEvent(event, detail), diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index 4817130f0a..b022c205a4 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -394,6 +394,7 @@ interface FrameStream { } interface FrameChunkResponse extends FrameChunk { + activePaths: string[]; coloring: Coloring; customizeColorSetting: CustomizeColor[]; colorscale: Colorscale; @@ -416,7 +417,9 @@ const createReader = ({ view, group, schema, + activePaths, }: { + activePaths: string[]; chunkSize: number; coloring: Coloring; customizeColorSetting: CustomizeColor[]; @@ -462,6 +465,7 @@ const createReader = ({ const { frames, range } = await call(); controller.enqueue({ + activePaths, frames, range, coloring, @@ -513,7 +517,8 @@ const getSendChunk = value.colorscale, value.labelTagColors, value.selectedLabelTags, - value.schema + value.schema, + value.activePaths ) ) ); @@ -561,6 +566,7 @@ const requestFrameChunk = ({ uuid }: RequestFrameChunk) => { }; interface SetStream { + activePaths: string[]; coloring: Coloring; customizeColorSetting: CustomizeColor[]; colorscale: Colorscale; @@ -592,6 +598,7 @@ const setStream = ({ view, group, schema, + activePaths, }: SetStream) => { stream && stream.cancel(); streamId = uuid; @@ -610,6 +617,7 @@ const setStream = ({ view, group, schema, + activePaths, }); stream.reader.read().then(getSendChunk(uuid)); From 767a32e4946a770decb632f13d7b32cb279cd74f Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 14:44:07 -0600 Subject: [PATCH 30/37] explicit type for lru color cache --- app/packages/looker/src/worker/resolve-color.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/looker/src/worker/resolve-color.ts b/app/packages/looker/src/worker/resolve-color.ts index 0e217ecbb8..d829e52913 100644 --- a/app/packages/looker/src/worker/resolve-color.ts +++ b/app/packages/looker/src/worker/resolve-color.ts @@ -12,7 +12,7 @@ export default (): [ (pool: string[], seed: number, key: string | number) => Promise, (result: ResolveColor) => void ] => { - const cache = new LRUCache({ + const cache = new LRUCache>({ max: MAX_COLOR_CACHE_SIZE, }); From 1c8d40029dce6516122dff971155ef0508791396 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 14:46:56 -0600 Subject: [PATCH 31/37] fix cache --- app/packages/looker/src/worker/resolve-color.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/packages/looker/src/worker/resolve-color.ts b/app/packages/looker/src/worker/resolve-color.ts index d829e52913..16c3147a7e 100644 --- a/app/packages/looker/src/worker/resolve-color.ts +++ b/app/packages/looker/src/worker/resolve-color.ts @@ -21,11 +21,12 @@ export default (): [ return [ (pool, seed, key) => { - if (!cache.has(seed)) { - cache[seed] = {}; - } + let colors = cache.get(seed); - const colors = cache[seed]; + if (!colors) { + colors = {}; + cache.set(seed, colors); + } if (!(key in colors)) { if (!(seed in requests)) { From c37a843fedad98a8256c7d3b22460afd180a93e0 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 9 Jan 2025 18:57:12 -0600 Subject: [PATCH 32/37] impl default visibility for frames --- app/packages/state/src/recoil/schema.ts | 97 ++++++++++++------------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/app/packages/state/src/recoil/schema.ts b/app/packages/state/src/recoil/schema.ts index 6f9f4e865f..a42a719065 100644 --- a/app/packages/state/src/recoil/schema.ts +++ b/app/packages/state/src/recoil/schema.ts @@ -322,64 +322,59 @@ export const dbPath = selectorFamily({ }, }); -export const defaultLabels = selectorFamily({ - key: "defaultLabels", - get: - ({ space }) => - ({ get }) => { - const schema = get(fieldSchema({ space })); - const denseLabels = getDenseLabelNames(schema); - const allLabels = get(labelFields({ space })); +export const defaultVisibleLabels = selector({ + key: "defaultVisibleLabels", + get: ({ get }) => { + const sampleSchema = get(fieldSchema({ space: State.SPACE.SAMPLE })); + const frameSchema = get(fieldSchema({ space: State.SPACE.FRAME })); - const defaultVisibleLabelsConfig = get(defaultVisibilityLabels); + const denseLabelsSamples = getDenseLabelNames(sampleSchema); + const denseLabelsFrames = getDenseLabelNames(frameSchema).map( + (l) => `frames.${l}` + ); - if ( - !defaultVisibleLabelsConfig?.include && - !defaultVisibleLabelsConfig?.exclude - ) { - // todo: frames - return allLabels.filter((label) => !denseLabels.includes(label)); - } + const denseLabels = [...denseLabelsSamples, ...denseLabelsFrames]; - if ( - defaultVisibleLabelsConfig.include && - !defaultVisibleLabelsConfig.exclude - ) { - return allLabels.filter((label) => - defaultVisibleLabelsConfig.include.includes(label) - ); - } + const allSampleLabels = get(labelFields({ space: State.SPACE.SAMPLE })); + const allFrameLabels = get(labelFields({ space: State.SPACE.FRAME })); + const allLabels = [...allSampleLabels, ...allFrameLabels]; - if ( - !defaultVisibleLabelsConfig.include && - defaultVisibleLabelsConfig.exclude - ) { - return allLabels.filter( - (label) => !defaultVisibleLabelsConfig.exclude.includes(label) - ); - } + const defaultVisibleLabelsConfig = get(defaultVisibilityLabels); - // is in both include and exclude - const includeList = new Set(defaultVisibleLabelsConfig.include); - const excludeList = new Set(defaultVisibleLabelsConfig.exclude); - // resolved = set(include) - set(exclude) - const resolved = new Set( - [...includeList].filter((x) => !excludeList.has(x)) + if ( + !defaultVisibleLabelsConfig?.include && + !defaultVisibleLabelsConfig?.exclude + ) { + return allLabels.filter((label) => !denseLabels.includes(label)); + } + + if ( + defaultVisibleLabelsConfig.include && + !defaultVisibleLabelsConfig.exclude + ) { + return allLabels.filter((label) => + defaultVisibleLabelsConfig.include.includes(label) ); - return allLabels.filter((label) => resolved.has(label)); + } - // todo: frames - if (space) { - return space === State.SPACE.FRAME - ? getLabelFields(get(atoms.frameFields), "frames.") - : getLabelFields(get(atoms.sampleFields)); - } + if ( + !defaultVisibleLabelsConfig.include && + defaultVisibleLabelsConfig.exclude + ) { + return allLabels.filter( + (label) => !defaultVisibleLabelsConfig.exclude.includes(label) + ); + } - return [ - ...getLabelFields(get(atoms.sampleFields)), - ...getLabelFields(get(atoms.frameFields), "frames."), - ]; - }, + // is in both include and exclude + const includeList = new Set(defaultVisibleLabelsConfig.include); + const excludeList = new Set(defaultVisibleLabelsConfig.exclude); + // resolved = set(include) - set(exclude) + const resolved = new Set( + [...includeList].filter((x) => !excludeList.has(x)) + ); + return allLabels.filter((label) => resolved.has(label)); + }, }); export const labelFields = selectorFamily({ @@ -575,7 +570,7 @@ export const activeFields = selectorFamily({ ({ modal }) => ({ get }) => { return filterPaths( - get(_activeFields({ modal })) || get(defaultLabels({})), + get(_activeFields({ modal })) || get(defaultVisibleLabels), buildSchema(get(atoms.sampleFields), get(atoms.frameFields)) ); }, From ebd99ae9ca50820a1acb75e8330c52f68385a191 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 10 Jan 2025 18:51:08 -0600 Subject: [PATCH 33/37] add sample guard in refreshSample --- app/packages/looker/src/lookers/abstract.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 093c270c35..ae5fae119b 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -534,7 +534,12 @@ export abstract class AbstractLooker< } refreshSample() { - this.updateSample(this.sample); + // todo: sometimes instance in spotlight?.updateItems() is defined but has no ref to sample + // this crashes the app. this is a bug and should be fixed + + if (this.sample) { + this.updateSample(this.sample); + } } getSample(): Promise { From 3975c8b8d36b52c747a43ee8a0a987f167288a84 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 10 Jan 2025 18:57:48 -0600 Subject: [PATCH 34/37] use more granular strategy for caching --- .../core/src/components/Grid/Grid.tsx | 25 ++++++- .../core/src/components/Grid/useRefreshers.ts | 21 ++---- .../core/src/components/Grid/useSelect.ts | 10 +++ .../Sidebar/useOnSidebarSelectionChange.ts | 68 ------------------- .../Sidebar/useShouldReloadSample.ts | 59 ++++++++++++++++ app/packages/looker/src/worker/index.ts | 1 + 6 files changed, 98 insertions(+), 86 deletions(-) delete mode 100644 app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts create mode 100644 app/packages/core/src/components/Sidebar/useShouldReloadSample.ts diff --git a/app/packages/core/src/components/Grid/Grid.tsx b/app/packages/core/src/components/Grid/Grid.tsx index 8948d8b1ff..e8209a1ff8 100644 --- a/app/packages/core/src/components/Grid/Grid.tsx +++ b/app/packages/core/src/components/Grid/Grid.tsx @@ -10,9 +10,10 @@ import React, { useRef, useState, } from "react"; -import { useRecoilValue } from "recoil"; +import { useRecoilCallback, useRecoilValue } from "recoil"; import { v4 as uuid } from "uuid"; import { QP_WAIT, QueryPerformanceToastEvent } from "../QueryPerformanceToast"; +import { gridActivePathsLUT } from "../Sidebar/useShouldReloadSample"; import { gridCrop, gridSpacing, pageParameters } from "./recoil"; import useAt from "./useAt"; import useEscape from "./useEscape"; @@ -46,6 +47,15 @@ function Grid() { const setSample = fos.useExpandSample(store); const getFontSize = useFontSize(id); + const getCurrentActiveLabelFields = useRecoilCallback( + ({ snapshot }) => + () => { + return snapshot + .getLoadable(fos.activeLabelFields({ modal: false })) + .getValue(); + } + ); + const spotlight = useMemo(() => { /** SPOTLIGHT REFRESHER */ reset; @@ -61,6 +71,7 @@ function Grid() { const looker = lookerStore.get(id.description); looker?.destroy(); lookerStore.delete(id.description); + gridActivePathsLUT.delete(id.description); }, detach: (id) => { const looker = lookerStore.get(id.description); @@ -101,6 +112,18 @@ function Grid() { ); lookerStore.set(id.description, looker); looker.attach(element, dimensions); + + // initialize active paths tracker + const currentActiveLabelFields = getCurrentActiveLabelFields(); + if ( + currentActiveLabelFields && + !gridActivePathsLUT.has(id.description) + ) { + gridActivePathsLUT.set( + id.description, + new Set(currentActiveLabelFields) + ); + } }, scrollbar: true, spacing, diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index c1cb65f362..23c88ec09c 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -1,10 +1,10 @@ import { subscribe } from "@fiftyone/relay"; import * as fos from "@fiftyone/state"; import { LRUCache } from "lru-cache"; -import { useEffect, useMemo, useRef } from "react"; +import { useEffect, useMemo } from "react"; import uuid from "react-uuid"; import { useRecoilValue } from "recoil"; -import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange"; +import { gridActivePathsLUT } from "../Sidebar/useShouldReloadSample"; import { gridAt, gridOffset, gridPage } from "./recoil"; const MAX_LRU_CACHE_ITEMS = 510; @@ -86,28 +86,15 @@ export default function useRefreshers() { /** LOOKER STORE REFRESHER */ return new LRUCache({ - dispose: (looker) => { + dispose: (looker, id) => { looker.destroy(); + gridActivePathsLUT.delete(id); }, max: MAX_LRU_CACHE_ITEMS, noDisposeOnSet: true, }); }, [reset]); - const lookerStoreRef = useRef(lookerStore); - lookerStoreRef.current = lookerStore; - - const gridLabelsToggleTracker = useOnSidebarSelectionChange({ modal: false }); - - /** - * Refresh all lookers when the labels toggle tracker changes - */ - useEffect(() => { - lookerStoreRef.current?.forEach((looker) => { - looker.refreshSample(); - }); - }, [gridLabelsToggleTracker]); - return { lookerStore, pageReset, diff --git a/app/packages/core/src/components/Grid/useSelect.ts b/app/packages/core/src/components/Grid/useSelect.ts index 7958c8b219..0619d46eea 100644 --- a/app/packages/core/src/components/Grid/useSelect.ts +++ b/app/packages/core/src/components/Grid/useSelect.ts @@ -3,6 +3,7 @@ import * as fos from "@fiftyone/state"; import type { LRUCache } from "lru-cache"; import { useEffect } from "react"; import { useRecoilValue } from "recoil"; +import { useShouldReloadSampleOnActiveFieldsChange } from "../Sidebar/useShouldReloadSample"; export default function useSelect( getFontSize: () => number, @@ -12,6 +13,10 @@ export default function useSelect( ) { const { init, deferred } = fos.useDeferrer(); + const shouldRefresh = useShouldReloadSampleOnActiveFieldsChange({ + modal: false, + }); + const selected = useRecoilValue(fos.selectedSamples); useEffect(() => { deferred(() => { @@ -29,6 +34,11 @@ export default function useSelect( fontSize, selected: selected.has(id.description), }); + + // rerender looker if active fields have changed and have never been rendered before + if (shouldRefresh(id.description)) { + instance.refreshSample(); + } }); for (const id of store.keys()) { diff --git a/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts deleted file mode 100644 index bea5539c33..0000000000 --- a/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { activeLabelFields } from "@fiftyone/state"; -import { useEffect, useRef } from "react"; -import { atomFamily, useRecoilState, useRecoilValue } from "recoil"; -import { gridPage } from "../Grid/recoil"; - -const labelsToggleTracker = atomFamily>, boolean>({ - key: "labelsToggleTracker", - default: new Map>(), -}); - -/** - * This hook is used to update the sidebar tracker when the user changes the - * selection of labels in the sidebar. We keep a map of grid page to the active - * label fields for that page. - */ -export const useOnSidebarSelectionChange = ({ modal }: { modal: boolean }) => { - const activeLabelFieldsValue = useRecoilValue(activeLabelFields({ modal })); - const gridPageValue = useRecoilValue(gridPage); - - const gridPageValueRef = useRef(gridPageValue); - - gridPageValueRef.current = gridPageValue; - - const [sidebarTracker, setSidebarTracker] = useRecoilState( - labelsToggleTracker(modal) - ); - - useEffect(() => { - const thisPageActiveFields = sidebarTracker.get(gridPageValue); - - const currentActiveLabelFields = new Set(activeLabelFieldsValue); - - if (currentActiveLabelFields.size === 0) { - return; - } - - // diff the two sets, we only care about net new fields - // if there are no new fields, we don't need to update the tracker - let hasNewFields = false; - if (thisPageActiveFields) { - for (const field of currentActiveLabelFields) { - if (!thisPageActiveFields.has(field)) { - hasNewFields = true; - break; - } - } - } else { - hasNewFields = true; - } - - if (!hasNewFields) { - return; - } - - const newTracker = new Map([ - ...sidebarTracker, - [gridPageValueRef.current, new Set(activeLabelFieldsValue)], - ]); - - if (newTracker.size === 0) { - return; - } - - setSidebarTracker(newTracker); - }, [activeLabelFieldsValue, sidebarTracker]); - - return sidebarTracker; -}; diff --git a/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts b/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts new file mode 100644 index 0000000000..41fdff7f90 --- /dev/null +++ b/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts @@ -0,0 +1,59 @@ +import { activeLabelFields } from "@fiftyone/state"; +import { useCallback } from "react"; +import { useRecoilValue } from "recoil"; + +type LookerId = string; +type CachedLabels = Set; + +export const gridActivePathsLUT = new Map(); + +export const useShouldReloadSampleOnActiveFieldsChange = ({ + modal, +}: { + modal: boolean; +}) => { + const activeLabelFieldsValue = useRecoilValue(activeLabelFields({ modal })); + + const shouldRefresh = useCallback( + (id: string) => { + const thisLookerActiveFields = gridActivePathsLUT.get(id); + const currentActiveLabelFields = new Set(activeLabelFieldsValue); + + if (currentActiveLabelFields.size === 0) { + return false; + } + + // diff the two sets, we only care about net new fields + // if there are no new fields, we don't need to update the tracker + let hasNewFields = false; + + if (thisLookerActiveFields) { + for (const field of currentActiveLabelFields) { + if (!thisLookerActiveFields.has(field)) { + hasNewFields = true; + break; + } + } + } else { + hasNewFields = true; + } + + if (!hasNewFields) { + return false; + } + + gridActivePathsLUT.set( + id, + new Set([ + ...(thisLookerActiveFields ?? []), + ...currentActiveLabelFields, + ]) + ); + + return true; + }, + [activeLabelFieldsValue] + ); + + return shouldRefresh; +}; diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index b022c205a4..0b054707b4 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -176,6 +176,7 @@ const processLabels = async ( activePaths.length && activePaths.includes(`${prefix ?? ""}${field}`) ) { + console.log(">>>painting ", `${prefix ?? ""}${field}`); if (painterFactory[cls]) { painterPromises.push( painterFactory[cls]( From 4630ad47a0da0087e49d2fa06a4273ee1ea39ad9 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 10 Jan 2025 20:17:54 -0600 Subject: [PATCH 35/37] move sync and check logic to another file and add unit tests --- .../core/src/components/Modal/Modal.tsx | 1 - .../Sidebar/syncAndCheckRefreshNeeded.test.ts | 96 +++++++++++++++++++ .../Sidebar/syncAndCheckRefreshNeeded.ts | 51 ++++++++++ .../Sidebar/useShouldReloadSample.ts | 41 ++------ 4 files changed, 153 insertions(+), 36 deletions(-) create mode 100644 app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.test.ts create mode 100644 app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts diff --git a/app/packages/core/src/components/Modal/Modal.tsx b/app/packages/core/src/components/Modal/Modal.tsx index bdfc8cbcb5..9ac42d1141 100644 --- a/app/packages/core/src/components/Modal/Modal.tsx +++ b/app/packages/core/src/components/Modal/Modal.tsx @@ -7,7 +7,6 @@ import { useRecoilCallback, useRecoilValue } from "recoil"; import styled from "styled-components"; import { ModalActionsRow } from "../Actions"; import Sidebar from "../Sidebar"; -import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange"; import ModalNavigation from "./ModalNavigation"; import { ModalSpace } from "./ModalSpace"; import { TooltipInfo } from "./TooltipInfo"; diff --git a/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.test.ts b/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.test.ts new file mode 100644 index 0000000000..7e92cdef5d --- /dev/null +++ b/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.test.ts @@ -0,0 +1,96 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { syncAndCheckRefreshNeeded } from "./syncAndCheckRefreshNeeded"; + +describe("syncAndCheckRefreshNeeded", () => { + let lut: Map>; + + beforeEach(() => { + lut = new Map(); + }); + + it("returns false if currentActiveLabelFields is empty", () => { + const id = "looker1"; + const currentActiveLabelFields = new Set(); + + const result = syncAndCheckRefreshNeeded(id, lut, currentActiveLabelFields); + expect(result).toBe(false); + // lut shoiuldn't be modified + expect(lut.has(id)).toBe(false); + }); + + it("returns true if lut is empty and there are active label fields", () => { + const id = "looker1"; + const currentActiveLabelFields = new Set(["segmentation1"]); + + const result = syncAndCheckRefreshNeeded(id, lut, currentActiveLabelFields); + expect(result).toBe(true); + + // lut should now have the newly added fields for looker1 + const storedFields = lut.get(id); + expect(storedFields).toEqual(new Set(["segmentation1"])); + }); + + it("returns true if the looker ID does not exist in lut (new looker)", () => { + lut.set("existingLooker", new Set(["seg1"])); + + const currentActiveLabelFields = new Set(["seg1", "heatmap1"]); + + const newLookerId = "newLooker"; + const result = syncAndCheckRefreshNeeded( + newLookerId, + lut, + currentActiveLabelFields + ); + expect(result).toBe(true); + + const storedFieldsNewLooker = lut.get(newLookerId); + expect(storedFieldsNewLooker).toEqual(new Set(["seg1", "heatmap1"])); + + const storedFieldsExistingLooker = lut.get("existingLooker"); + expect(storedFieldsExistingLooker).toEqual(new Set(["seg1"])); + }); + + it("returns false if lut already has the same fields", () => { + const id = "looker2"; + lut.set(id, new Set(["seg1", "seg2"])); + + const currentActiveLabelFields = new Set(["seg1", "seg2"]); + const result = syncAndCheckRefreshNeeded(id, lut, currentActiveLabelFields); + + expect(result).toBe(false); + // lut remains the same, no changes + expect(lut.get(id)).toEqual(new Set(["seg1", "seg2"])); + }); + + it("returns true if lut has a subset of the current fields (some new ones are missing)", () => { + const id = "looker2"; + // lut only has "field1" + lut.set(id, new Set(["field1"])); + + // incoming fields: "field1" (already known) and "field2" (new) + const currentActiveLabelFields = new Set(["field1", "field2"]); + const result = syncAndCheckRefreshNeeded(id, lut, currentActiveLabelFields); + + expect(result).toBe(true); + + // lut should have been updated to include both fields + expect(lut.get(id)).toEqual(new Set(["field1", "field2"])); + }); + + it("does not keep returning true after lut is updated", () => { + const id = "looker3"; + lut.set(id, new Set(["field1"])); + // This call should add "field2" + let result = syncAndCheckRefreshNeeded( + id, + lut, + new Set(["field1", "field2"]) + ); + expect(result).toBe(true); + expect(lut.get(id)).toEqual(new Set(["field1", "field2"])); + + // second call with the same fields should now return false + result = syncAndCheckRefreshNeeded(id, lut, new Set(["field1", "field2"])); + expect(result).toBe(false); + }); +}); diff --git a/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts b/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts new file mode 100644 index 0000000000..ca8946311d --- /dev/null +++ b/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts @@ -0,0 +1,51 @@ +import { CachedLabels, LookerId } from "./useShouldReloadSample"; + +/** + * Checks if a given looker needs to be refreshed based on active label fields. + * + * @param lookerId - unique Looker ID + * @param lut - look-up table tracking which fields each looker has already rendered at least once (and therefore cached) + * @param currentActiveLabelFields - set of active label fields for the modal/looker + * + * @returns whether or not looker should be refreshed (true) or not (false) + */ +export function syncAndCheckRefreshNeeded( + lookerId: string, + lut: Map, + currentActiveLabelFields: Set +): boolean { + // if no active fields, no refresh is needed + if (currentActiveLabelFields.size === 0) { + return false; + } + + const thisLookerActiveFields = lut.get(lookerId); + + // we only care about net-new fields. + let hasNewFields = false; + + if (thisLookerActiveFields) { + for (const field of currentActiveLabelFields) { + if (!thisLookerActiveFields.has(field)) { + hasNewFields = true; + break; + } + } + } else { + // if `thisLookerActiveFields` is undefined, it means this looker has not been considered yet + hasNewFields = true; + } + + // if no new fields, then no refresh is needed + if (!hasNewFields) { + return false; + } + + // update cached labels set for this looker + lut.set( + lookerId, + new Set([...(thisLookerActiveFields ?? []), ...currentActiveLabelFields]) + ); + + return true; +} diff --git a/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts b/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts index 41fdff7f90..3d3784706c 100644 --- a/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts +++ b/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts @@ -1,9 +1,10 @@ import { activeLabelFields } from "@fiftyone/state"; import { useCallback } from "react"; import { useRecoilValue } from "recoil"; +import { syncAndCheckRefreshNeeded } from "./syncAndCheckRefreshNeeded"; -type LookerId = string; -type CachedLabels = Set; +export type LookerId = string; +export type CachedLabels = Set; export const gridActivePathsLUT = new Map(); @@ -16,41 +17,11 @@ export const useShouldReloadSampleOnActiveFieldsChange = ({ const shouldRefresh = useCallback( (id: string) => { - const thisLookerActiveFields = gridActivePathsLUT.get(id); - const currentActiveLabelFields = new Set(activeLabelFieldsValue); - - if (currentActiveLabelFields.size === 0) { - return false; - } - - // diff the two sets, we only care about net new fields - // if there are no new fields, we don't need to update the tracker - let hasNewFields = false; - - if (thisLookerActiveFields) { - for (const field of currentActiveLabelFields) { - if (!thisLookerActiveFields.has(field)) { - hasNewFields = true; - break; - } - } - } else { - hasNewFields = true; - } - - if (!hasNewFields) { - return false; - } - - gridActivePathsLUT.set( + return syncAndCheckRefreshNeeded( id, - new Set([ - ...(thisLookerActiveFields ?? []), - ...currentActiveLabelFields, - ]) + gridActivePathsLUT, + new Set(activeLabelFieldsValue) ); - - return true; }, [activeLabelFieldsValue] ); From acbf069df257034a8169fd52c8458c69e05cbf84 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 10 Jan 2025 20:51:51 -0600 Subject: [PATCH 36/37] remove debug log --- app/packages/looker/src/worker/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/packages/looker/src/worker/index.ts b/app/packages/looker/src/worker/index.ts index 0b054707b4..b022c205a4 100644 --- a/app/packages/looker/src/worker/index.ts +++ b/app/packages/looker/src/worker/index.ts @@ -176,7 +176,6 @@ const processLabels = async ( activePaths.length && activePaths.includes(`${prefix ?? ""}${field}`) ) { - console.log(">>>painting ", `${prefix ?? ""}${field}`); if (painterFactory[cls]) { painterPromises.push( painterFactory[cls]( From d8ecb444826ea0df1678b65d50c0b265f071f13e Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Fri, 10 Jan 2025 21:29:02 -0600 Subject: [PATCH 37/37] fix modal --- .../core/src/components/Modal/Modal.tsx | 8 +------- .../core/src/components/Modal/use-looker.ts | 15 +++++++++++++++ .../Sidebar/syncAndCheckRefreshNeeded.ts | 6 +++--- .../Sidebar/useShouldReloadSample.ts | 18 ++++++++++++++++-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/app/packages/core/src/components/Modal/Modal.tsx b/app/packages/core/src/components/Modal/Modal.tsx index 9ac42d1141..6443fa8a2b 100644 --- a/app/packages/core/src/components/Modal/Modal.tsx +++ b/app/packages/core/src/components/Modal/Modal.tsx @@ -1,7 +1,7 @@ import { HelpPanel, JSONPanel } from "@fiftyone/components"; import { OPERATOR_PROMPT_AREAS, OperatorPromptArea } from "@fiftyone/operators"; import * as fos from "@fiftyone/state"; -import React, { useCallback, useEffect, useMemo, useRef } from "react"; +import React, { useCallback, useMemo, useRef } from "react"; import ReactDOM from "react-dom"; import { useRecoilCallback, useRecoilValue } from "recoil"; import styled from "styled-components"; @@ -178,12 +178,6 @@ const Modal = () => { const activeLookerRef = useRef(); - const modalLabelsToggleTracker = useOnSidebarSelectionChange({ modal: true }); - - useEffect(() => { - activeLookerRef.current?.refreshSample(); - }, [modalLabelsToggleTracker]); - const addTooltipEventHandler = useTooltipEventHandler(); const removeTooltipEventHanlderRef = useRef({ !initialRef.current && looker.updateOptions(lookerOptions); }, [looker, lookerOptions]); + const shouldRefresh = useShouldReloadSampleOnActiveFieldsChange({ + modal: true, + }); + + useEffect(() => { + if (!looker) { + return; + } + + if (shouldRefresh(id)) { + looker?.refreshSample(); + } + }, [id, lookerOptions.activePaths, looker]); + useEffect(() => { /** start refreshers */ colorScheme; diff --git a/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts b/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts index ca8946311d..c909da55ac 100644 --- a/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts +++ b/app/packages/core/src/components/Sidebar/syncAndCheckRefreshNeeded.ts @@ -9,11 +9,11 @@ import { CachedLabels, LookerId } from "./useShouldReloadSample"; * * @returns whether or not looker should be refreshed (true) or not (false) */ -export function syncAndCheckRefreshNeeded( +export const syncAndCheckRefreshNeeded = ( lookerId: string, lut: Map, currentActiveLabelFields: Set -): boolean { +) => { // if no active fields, no refresh is needed if (currentActiveLabelFields.size === 0) { return false; @@ -48,4 +48,4 @@ export function syncAndCheckRefreshNeeded( ); return true; -} +}; diff --git a/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts b/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts index 3d3784706c..2baaecb8f7 100644 --- a/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts +++ b/app/packages/core/src/components/Sidebar/useShouldReloadSample.ts @@ -1,5 +1,5 @@ import { activeLabelFields } from "@fiftyone/state"; -import { useCallback } from "react"; +import { useCallback, useEffect } from "react"; import { useRecoilValue } from "recoil"; import { syncAndCheckRefreshNeeded } from "./syncAndCheckRefreshNeeded"; @@ -7,6 +7,7 @@ export type LookerId = string; export type CachedLabels = Set; export const gridActivePathsLUT = new Map(); +export const modalActivePathsLUT = new Map(); export const useShouldReloadSampleOnActiveFieldsChange = ({ modal, @@ -19,12 +20,25 @@ export const useShouldReloadSampleOnActiveFieldsChange = ({ (id: string) => { return syncAndCheckRefreshNeeded( id, - gridActivePathsLUT, + modal ? modalActivePathsLUT : gridActivePathsLUT, new Set(activeLabelFieldsValue) ); }, [activeLabelFieldsValue] ); + /** + * clear look up table when component unmounts + */ + useEffect(() => { + return () => { + if (modal) { + modalActivePathsLUT.clear(); + } else { + gridActivePathsLUT.clear(); + } + }; + }, []); + return shouldRefresh; };