diff --git a/web/src/lib/components/faces-page/manage-people-visibility.svelte b/web/src/lib/components/faces-page/manage-people-visibility.svelte
index a8af0522ab..749bae5d4d 100644
--- a/web/src/lib/components/faces-page/manage-people-visibility.svelte
+++ b/web/src/lib/components/faces-page/manage-people-visibility.svelte
@@ -10,6 +10,7 @@
   import { Button, IconButton, toastManager } from '@immich/ui';
   import { mdiClose, mdiEye, mdiEyeOff, mdiEyeSettings, mdiRestart } from '@mdi/js';
   import { t } from 'svelte-i18n';
+  import { SvelteMap } from 'svelte/reactivity';
 
   interface Props {
     people: PersonResponseDto[];
@@ -23,14 +24,7 @@
 
   let toggleVisibility = $state(ToggleVisibility.SHOW_ALL);
   let showLoadingSpinner = $state(false);
-
-  const getPersonIsHidden = (people: PersonResponseDto[]) => {
-    const personIsHidden: Record<string, boolean> = {};
-    for (const person of people) {
-      personIsHidden[person.id] = person.isHidden;
-    }
-    return personIsHidden;
-  };
+  const overrides = new SvelteMap<string, boolean>();
 
   const getNextVisibility = (toggleVisibility: ToggleVisibility) => {
     if (toggleVisibility === ToggleVisibility.SHOW_ALL) {
@@ -46,23 +40,23 @@
     toggleVisibility = getNextVisibility(toggleVisibility);
 
     for (const person of people) {
+      let isHidden = overrides.get(person.id) ?? person.isHidden;
+
       if (toggleVisibility === ToggleVisibility.HIDE_ALL) {
-        personIsHidden[person.id] = true;
+        isHidden = true;
       } else if (toggleVisibility === ToggleVisibility.SHOW_ALL) {
-        personIsHidden[person.id] = false;
+        isHidden = false;
       } else if (toggleVisibility === ToggleVisibility.HIDE_UNNANEMD && !person.name) {
-        personIsHidden[person.id] = true;
+        isHidden = true;
       }
+
+      setHiddenOverride(person, isHidden);
     }
   };
 
-  const handleResetVisibility = () => (personIsHidden = getPersonIsHidden(people));
-
   const handleSaveVisibility = async () => {
     showLoadingSpinner = true;
-    const changed = people
-      .filter((person) => person.isHidden !== personIsHidden[person.id])
-      .map((person) => ({ id: person.id, isHidden: personIsHidden[person.id] }));
+    const changed = Array.from(overrides, ([id, isHidden]) => ({ id, isHidden }));
 
     try {
       if (changed.length > 0) {
@@ -76,8 +70,12 @@
       }
 
       for (const person of people) {
-        person.isHidden = personIsHidden[person.id];
+        const isHidden = overrides.get(person.id);
+        if (isHidden !== undefined) {
+          person.isHidden = isHidden;
+        }
       }
+      overrides.clear();
 
       onClose();
     } catch (error) {
@@ -87,15 +85,13 @@
     }
   };
 
-  let personIsHidden = $state<Record<string, boolean>>({});
-
-  $effect(() => {
-    for (const person of people) {
-      if (!(person.id in personIsHidden)) {
-        personIsHidden[person.id] = person.isHidden;
-      }
+  const setHiddenOverride = (person: PersonResponseDto, isHidden: boolean) => {
+    if (isHidden === person.isHidden) {
+      overrides.delete(person.id);
+      return;
     }
-  });
+    overrides.set(person.id, isHidden);
+  };
 
   let toggleButtonOptions: Record<ToggleVisibility, { icon: string; label: string }> = $derived({
     [ToggleVisibility.HIDE_ALL]: { icon: mdiEyeOff, label: $t('hide_all_people') },
@@ -132,7 +128,7 @@
         variant="ghost"
         aria-label={$t('reset_people_visibility')}
         icon={mdiRestart}
-        onclick={handleResetVisibility}
+        onclick={() => overrides.clear()}
       />
       <IconButton
         shape="round"
@@ -150,11 +146,11 @@
 <div class="flex flex-wrap gap-1 p-2 pb-8 md:px-8 mt-16">
   <PeopleInfiniteScroll {people} hasNextPage={true} {loadNextPage}>
     {#snippet children({ person })}
-      {@const hidden = personIsHidden[person.id]}
+      {@const hidden = overrides.get(person.id) ?? person.isHidden}
       <button
         type="button"
         class="group relative w-full h-full"
-        onclick={() => (personIsHidden[person.id] = !hidden)}
+        onclick={() => setHiddenOverride(person, !hidden)}
         aria-pressed={hidden}
         aria-label={person.name ? $t('hide_named_person', { values: { name: person.name } }) : $t('hide_person')}
       >
diff --git a/web/src/routes/(user)/people/+page.svelte b/web/src/routes/(user)/people/+page.svelte
index a6f9e475e0..b051b44a2c 100644
--- a/web/src/routes/(user)/people/+page.svelte
+++ b/web/src/routes/(user)/people/+page.svelte
@@ -213,7 +213,11 @@
     await clearQueryParam(QueryParameter.SEARCHED_PEOPLE, $page.url);
   };
 
-  let people = $derived(data.people.people);
+  let people = $derived.by(() => {
+    // make deeply reactive
+    const _ = $state(data.people.people);
+    return _;
+  });
   let visiblePeople = $derived(people.filter((people) => !people.isHidden));
   let countVisiblePeople = $derived(searchName ? searchedPeopleLocal.length : data.people.total - data.people.hidden);
   let showPeople = $derived(searchName ? searchedPeopleLocal : visiblePeople);