feat(server): refresh face detection (#12335)

* refresh faces

handle non-ml faces

* fix metadata face handling

* updated tests

* added todo comment

server/src/services/person.service.ts

@@ -21,6 +21,7 @@ import {
 } from 'src/dtos/person.dto';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
+import { FaceSearchEntity } from 'src/entities/face-search.entity';
 import { PersonEntity } from 'src/entities/person.entity';
 import {
   AssetType,
@@ -256,14 +257,14 @@ export class PersonService extends BaseService {
     }
 
     const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
-      return force
-        ? this.assetRepository.getAll(pagination, {
+      return force === false
+        ? this.assetRepository.getWithout(pagination, WithoutProperty.FACES)
+        : this.assetRepository.getAll(pagination, {
             orderDirection: 'DESC',
             withFaces: true,
             withArchived: true,
             isVisible: true,
-          })
-        : this.assetRepository.getWithout(pagination, WithoutProperty.FACES);
+          });
     });
 
     for await (const assets of assetPagination) {
@@ -272,6 +273,10 @@ export class PersonService extends BaseService {
       );
     }
 
+    if (force === undefined) {
+      await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
+    }
+
     return JobStatus.SUCCESS;
   }
 
@@ -290,11 +295,11 @@ export class PersonService extends BaseService {
     };
     const [asset] = await this.assetRepository.getByIds([id], relations);
     const { previewFile } = getAssetFiles(asset.files);
-    if (!asset || !previewFile || asset.faces?.length > 0) {
+    if (!asset || !previewFile) {
       return JobStatus.FAILED;
     }
 
-    if (!asset.isVisible || asset.faces.length > 0) {
+    if (!asset.isVisible) {
       return JobStatus.SKIPPED;
     }
 
@@ -303,39 +308,82 @@ export class PersonService extends BaseService {
       previewFile.path,
       machineLearning.facialRecognition,
     );
-
     this.logger.debug(`${faces.length} faces detected in ${previewFile.path}`);
 
-    if (faces.length > 0) {
-      await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } });
-      const mappedFaces: Partial<AssetFaceEntity>[] = [];
-      for (const face of faces) {
+    const facesToAdd: (Partial<AssetFaceEntity> & { id: string })[] = [];
+    const embeddings: FaceSearchEntity[] = [];
+    const mlFaceIds = new Set<string>();
+    for (const face of asset.faces) {
+      if (face.sourceType === SourceType.MACHINE_LEARNING) {
+        mlFaceIds.add(face.id);
+      }
+    }
+
+    const heightScale = imageHeight / (asset.faces[0]?.imageHeight || 1);
+    const widthScale = imageWidth / (asset.faces[0]?.imageWidth || 1);
+    for (const { boundingBox, embedding } of faces) {
+      const scaledBox = {
+        x1: boundingBox.x1 * widthScale,
+        y1: boundingBox.y1 * heightScale,
+        x2: boundingBox.x2 * widthScale,
+        y2: boundingBox.y2 * heightScale,
+      };
+      const match = asset.faces.find((face) => this.iou(face, scaledBox) > 0.5);
+
+      if (match && !mlFaceIds.delete(match.id)) {
+        embeddings.push({ faceId: match.id, embedding });
+      } else {
         const faceId = this.cryptoRepository.randomUUID();
-        mappedFaces.push({
+        facesToAdd.push({
           id: faceId,
           assetId: asset.id,
           imageHeight,
           imageWidth,
-          boundingBoxX1: face.boundingBox.x1,
-          boundingBoxY1: face.boundingBox.y1,
-          boundingBoxX2: face.boundingBox.x2,
-          boundingBoxY2: face.boundingBox.y2,
-          faceSearch: { faceId, embedding: face.embedding },
+          boundingBoxX1: boundingBox.x1,
+          boundingBoxY1: boundingBox.y1,
+          boundingBoxX2: boundingBox.x2,
+          boundingBoxY2: boundingBox.y2,
         });
+        embeddings.push({ faceId, embedding });
       }
+    }
+    const faceIdsToRemove = [...mlFaceIds];
 
-      const faceIds = await this.personRepository.createFaces(mappedFaces);
-      await this.jobRepository.queueAll(faceIds.map((id) => ({ name: JobName.FACIAL_RECOGNITION, data: { id } })));
+    if (facesToAdd.length > 0 || faceIdsToRemove.length > 0 || embeddings.length > 0) {
+      await this.personRepository.refreshFaces(facesToAdd, faceIdsToRemove, embeddings);
     }
 
-    await this.assetRepository.upsertJobStatus({
-      assetId: asset.id,
-      facesRecognizedAt: new Date(),
-    });
+    if (faceIdsToRemove.length > 0) {
+      this.logger.log(`Removed ${faceIdsToRemove.length} faces below detection threshold in asset ${id}`);
+    }
+
+    if (facesToAdd.length > 0) {
+      this.logger.log(`Detected ${facesToAdd.length} new faces in asset ${id}`);
+      const jobs = facesToAdd.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id } }) as const);
+      await this.jobRepository.queueAll([{ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }, ...jobs]);
+    } else if (embeddings.length > 0) {
+      this.logger.log(`Added ${embeddings.length} face embeddings for asset ${id}`);
+    }
+
+    await this.assetRepository.upsertJobStatus({ assetId: asset.id, facesRecognizedAt: new Date() });
 
     return JobStatus.SUCCESS;
   }
 
+  private iou(face: AssetFaceEntity, newBox: BoundingBox): number {
+    const x1 = Math.max(face.boundingBoxX1, newBox.x1);
+    const y1 = Math.max(face.boundingBoxY1, newBox.y1);
+    const x2 = Math.min(face.boundingBoxX2, newBox.x2);
+    const y2 = Math.min(face.boundingBoxY2, newBox.y2);
+
+    const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
+    const area1 = (face.boundingBoxX2 - face.boundingBoxX1) * (face.boundingBoxY2 - face.boundingBoxY1);
+    const area2 = (newBox.x2 - newBox.x1) * (newBox.y2 - newBox.y1);
+    const union = area1 + area2 - intersection;
+
+    return intersection / union;
+  }
+
   async handleQueueRecognizeFaces({ force, nightly }: INightlyJob): Promise<JobStatus> {
     const { machineLearning } = await this.getConfig({ withCache: false });
     if (!isFacialRecognitionEnabled(machineLearning)) {
@@ -371,7 +419,7 @@ export class PersonService extends BaseService {
     const lastRun = new Date().toISOString();
     const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
       this.personRepository.getAllFaces(pagination, {
-        where: force ? undefined : { personId: IsNull(), sourceType: IsNull() },
+        where: force ? undefined : { personId: IsNull(), sourceType: SourceType.MACHINE_LEARNING },
       }),
     );