diff --git a/app/packages/core/src/components/Grid/Grid.tsx b/app/packages/core/src/components/Grid/Grid.tsx
index 8948d8b1ff..c45cfb8507 100644
--- a/app/packages/core/src/components/Grid/Grid.tsx
+++ b/app/packages/core/src/components/Grid/Grid.tsx
@@ -13,6 +13,7 @@ import React, {
 import { useRecoilValue } from "recoil";
 import { v4 as uuid } from "uuid";
 import { QP_WAIT, QueryPerformanceToastEvent } from "../QueryPerformanceToast";
+import { useOnSidebarSelectionChange } from "../Sidebar/useOnSidebarSelectionChange";
 import { gridCrop, gridSpacing, pageParameters } from "./recoil";
 import useAt from "./useAt";
 import useEscape from "./useEscape";
@@ -46,6 +47,8 @@ function Grid() {
   const setSample = fos.useExpandSample(store);
   const getFontSize = useFontSize(id);
 
+  useOnSidebarSelectionChange();
+
   const spotlight = useMemo(() => {
     /** SPOTLIGHT REFRESHER */
     reset;
diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts
index 1fa1b38d09..d980092dc3 100644
--- a/app/packages/core/src/components/Grid/useRefreshers.ts
+++ b/app/packages/core/src/components/Grid/useRefreshers.ts
@@ -29,14 +29,23 @@ export default function useRefreshers() {
   );
   const view = fos.filterView(useRecoilValue(fos.view));
 
+  const labelsToggleTracker = useRecoilValue(fos.labelsToggleTracker);
+
   // only reload, attempt to return to the last grid location
   const layoutReset = useMemo(() => {
     cropToContent;
     fieldVisibilityStage;
+    labelsToggleTracker;
     mediaField;
     refresher;
     return uuid();
-  }, [cropToContent, fieldVisibilityStage, mediaField, refresher]);
+  }, [
+    cropToContent,
+    fieldVisibilityStage,
+    labelsToggleTracker,
+    mediaField,
+    refresher,
+  ]);
 
   // the values reset the page, i.e. return to the top
   const pageReset = useMemo(() => {
@@ -64,18 +73,20 @@ export default function useRefreshers() {
     return uuid();
   }, [layoutReset, pageReset]);
 
-  useEffect(
-    () =>
-      subscribe(({ event }, { reset }) => {
-        if (event === "fieldVisibility") return;
+  useEffect(() => {
+    const unsubscribe = subscribe(({ event }, { reset }) => {
+      if (event === "fieldVisibility") return;
 
-        // if not a modal page change, reset the grid location
-        reset(gridAt);
-        reset(gridPage);
-        reset(gridOffset);
-      }),
-    []
-  );
+      // if not a modal page change, reset the grid location
+      reset(gridAt);
+      reset(gridPage);
+      reset(gridOffset);
+    });
+
+    return () => {
+      unsubscribe();
+    };
+  }, []);
 
   const lookerStore = useMemo(() => {
     /** LOOKER STORE REFRESHER */
diff --git a/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts
new file mode 100644
index 0000000000..f74d773e86
--- /dev/null
+++ b/app/packages/core/src/components/Sidebar/useOnSidebarSelectionChange.ts
@@ -0,0 +1,63 @@
+import { activeLabelFields, labelsToggleTracker } from "@fiftyone/state";
+import { useEffect, useRef } from "react";
+import { useRecoilValue, useSetRecoilState } from "recoil";
+import { gridPage } from "../Grid/recoil";
+
+/**
+ * This hook is used to update the sidebar tracker when the user changes the
+ * selection of labels in the sidebar. We keep a map of grid page to the active
+ * label fields for that page.
+ */
+export const useOnSidebarSelectionChange = () => {
+  const activeLabelFieldsValue = useRecoilValue(
+    activeLabelFields({ modal: false })
+  );
+  const gridPageValue = useRecoilValue(gridPage);
+
+  const gridPageValueRef = useRef(gridPageValue);
+
+  gridPageValueRef.current = gridPageValue;
+
+  const setSidebarTracker = useSetRecoilState(labelsToggleTracker);
+
+  const debugSidebarTracker = useRecoilValue(labelsToggleTracker);
+
+  useEffect(() => {
+    const thisPageActiveFields = debugSidebarTracker.get(gridPageValue);
+
+    const currentActiveLabelFields = new Set(activeLabelFieldsValue);
+
+    if (currentActiveLabelFields.size === 0) {
+      return;
+    }
+
+    // diff the two sets, we only care about net new fields
+    // if there are no new fields, we don't need to update the tracker
+    let hasNewFields = false;
+    if (thisPageActiveFields) {
+      for (const field of currentActiveLabelFields) {
+        if (!thisPageActiveFields.has(field)) {
+          hasNewFields = true;
+          break;
+        }
+      }
+    } else {
+      hasNewFields = true;
+    }
+
+    if (!hasNewFields) {
+      return;
+    }
+
+    const newTracker = new Map([
+      ...debugSidebarTracker,
+      [gridPageValueRef.current, new Set(activeLabelFieldsValue)],
+    ]);
+
+    if (newTracker.size === 0) {
+      return;
+    }
+
+    setSidebarTracker(newTracker);
+  }, [activeLabelFieldsValue, debugSidebarTracker]);
+};
diff --git a/app/packages/state/src/recoil/sidebar.ts b/app/packages/state/src/recoil/sidebar.ts
index d92783dbdd..e3d8f4d002 100644
--- a/app/packages/state/src/recoil/sidebar.ts
+++ b/app/packages/state/src/recoil/sidebar.ts
@@ -29,6 +29,7 @@ import type { VariablesOf } from "react-relay";
 import { commitMutation } from "react-relay";
 import {
   DefaultValue,
+  atom,
   atomFamily,
   selector,
   selectorFamily,
@@ -72,6 +73,9 @@ import {
 } from "./utils";
 import * as viewAtoms from "./view";
 
+type GridPageNumber = number;
+type SidebarEntriesSet = Set<string>;
+
 export enum EntryKind {
   EMPTY = "EMPTY",
   GROUP = "GROUP",
@@ -906,6 +910,11 @@ export const groupShown = selectorFamily<
     },
 });
 
+export const labelsToggleTracker = atom({
+  key: "labelsToggleTracker",
+  default: new Map<GridPageNumber, SidebarEntriesSet>(),
+});
+
 export const textFilter = atomFamily<string, boolean>({
   key: "textFilter",
   default: "",