feat: ocr (#18836)

* feat: add OCR functionality and related configurations

* chore: update labeler configuration for machine learning files

* feat(i18n): enhance OCR model descriptions and add orientation classification and unwarping features

* chore: update Dockerfile to include ccache for improved build performance

* feat(ocr): enhance OCR model configuration with orientation classification and unwarping options, update PaddleOCR integration, and improve response structure

* refactor(ocr): remove OCR_CLEANUP job from enum and type definitions

* refactor(ocr): remove obsolete OCR entity and migration files, and update asset job status and schema to accommodate new OCR table structure

* refactor(ocr): update OCR schema and response structure to use individual coordinates instead of bounding box, and adjust related service and repository files

* feat: enhance OCR configuration and functionality

- Updated OCR settings to include minimum detection box score, minimum detection score, and minimum recognition score.
- Refactored PaddleOCRecognizer to utilize new scoring parameters.
- Introduced new database tables for asset OCR data and search functionality.
- Modified related services and repositories to support the new OCR features.
- Updated translations for improved clarity in settings UI.

* sql changes

* use rapidocr

* change dto

* update web

* update lock

* update api

* store positions as normalized floats

* match column order in db

* update admin ui settings descriptions

fix max resolution key

set min threshold to 0.1

fix bind

* apply config correctly, adjust defaults

* unnecessary model type

* unnecessary sources

* fix(ocr): switch RapidOCR lang type from LangDet to LangRec

* fix(ocr): expose lang_type (LangRec.CH) and font_path on OcrOptions for RapidOCR

* fix(ocr): make OCR text search case- and accent-insensitive using ILIKE + unaccent

* fix(ocr): add OCR search fields

* fix: Add OCR database migration and update ML prediction logic.

* trigrams are already case insensitive

* add tests

* format

* update migrations

* wrong uuid function

* linting

* maybe fix medium tests

* formatting

* fix weblate check

* openapi

* sql

* minor fixes

* maybe fix medium tests part 2

* passing medium tests

* format web

* readd sql

* format dart

* disabled in e2e

* chore: translation ordering

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/repositories/asset-job.repository.ts

@@ -16,6 +16,7 @@ import {
   withExifInner,
   withFaces,
   withFacesAndPeople,
+  withFilePath,
   withFiles,
 } from 'src/utils/database';
 
@@ -192,6 +193,15 @@ export class AssetJobRepository {
       .executeTakeFirst();
   }
 
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getForOcr(id: string) {
+    return this.db
+      .selectFrom('asset')
+      .select((eb) => ['asset.visibility', withFilePath(eb, AssetFileType.Preview).as('previewFile')])
+      .where('asset.id', '=', id)
+      .executeTakeFirst();
+  }
+
   @GenerateSql({ params: [[DummyValue.UUID]] })
   getForSyncAssets(ids: string[]) {
     return this.db
@@ -348,6 +358,21 @@ export class AssetJobRepository {
       .stream();
   }
 
+  @GenerateSql({ params: [], stream: true })
+  streamForOcrJob(force?: boolean) {
+    return this.db
+      .selectFrom('asset')
+      .select(['asset.id'])
+      .$if(!force, (qb) =>
+        qb
+          .innerJoin('asset_job_status', 'asset_job_status.assetId', 'asset.id')
+          .where('asset_job_status.ocrAt', 'is', null),
+      )
+      .where('asset.deletedAt', 'is', null)
+      .where('asset.visibility', '!=', AssetVisibility.Hidden)
+      .stream();
+  }
+
   @GenerateSql({ params: [DummyValue.DATE], stream: true })
   streamForMigrationJob() {
     return this.db.selectFrom('asset').select(['id']).where('asset.deletedAt', 'is', null).stream();

server/src/repositories/asset.repository.ts

@@ -205,6 +205,7 @@ export class AssetRepository {
               metadataExtractedAt: eb.ref('excluded.metadataExtractedAt'),
               previewAt: eb.ref('excluded.previewAt'),
               thumbnailAt: eb.ref('excluded.thumbnailAt'),
+              ocrAt: eb.ref('excluded.ocrAt'),
             },
             values[0],
           ),

server/src/repositories/index.ts

@@ -25,6 +25,7 @@ import { MetadataRepository } from 'src/repositories/metadata.repository';
 import { MoveRepository } from 'src/repositories/move.repository';
 import { NotificationRepository } from 'src/repositories/notification.repository';
 import { OAuthRepository } from 'src/repositories/oauth.repository';
+import { OcrRepository } from 'src/repositories/ocr.repository';
 import { PartnerRepository } from 'src/repositories/partner.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { ProcessRepository } from 'src/repositories/process.repository';
@@ -74,6 +75,7 @@ export const repositories = [
   MoveRepository,
   NotificationRepository,
   OAuthRepository,
+  OcrRepository,
   PartnerRepository,
   PersonRepository,
   ProcessRepository,

server/src/repositories/machine-learning.repository.ts

@@ -15,6 +15,7 @@ export interface BoundingBox {
 export enum ModelTask {
   FACIAL_RECOGNITION = 'facial-recognition',
   SEARCH = 'clip',
+  OCR = 'ocr',
 }
 
 export enum ModelType {
@@ -23,6 +24,7 @@ export enum ModelType {
   RECOGNITION = 'recognition',
   TEXTUAL = 'textual',
   VISUAL = 'visual',
+  OCR = 'ocr',
 }
 
 export type ModelPayload = { imagePath: string } | { text: string };
@@ -30,7 +32,11 @@ export type ModelPayload = { imagePath: string } | { text: string };
 type ModelOptions = { modelName: string };
 
 export type FaceDetectionOptions = ModelOptions & { minScore: number };
-
+export type OcrOptions = ModelOptions & {
+  minDetectionScore: number;
+  minRecognitionScore: number;
+  maxResolution: number;
+};
 type VisualResponse = { imageHeight: number; imageWidth: number };
 export type ClipVisualRequest = { [ModelTask.SEARCH]: { [ModelType.VISUAL]: ModelOptions } };
 export type ClipVisualResponse = { [ModelTask.SEARCH]: string } & VisualResponse;
@@ -38,6 +44,21 @@ export type ClipVisualResponse = { [ModelTask.SEARCH]: string } & VisualResponse
 export type ClipTextualRequest = { [ModelTask.SEARCH]: { [ModelType.TEXTUAL]: ModelOptions } };
 export type ClipTextualResponse = { [ModelTask.SEARCH]: string };
 
+export type OCR = {
+  text: string[];
+  box: number[];
+  boxScore: number[];
+  textScore: number[];
+};
+
+export type OcrRequest = {
+  [ModelTask.OCR]: {
+    [ModelType.DETECTION]: ModelOptions & { options: { minScore: number; maxResolution: number } };
+    [ModelType.RECOGNITION]: ModelOptions & { options: { minScore: number } };
+  };
+};
+export type OcrResponse = { [ModelTask.OCR]: OCR } & VisualResponse;
+
 export type FacialRecognitionRequest = {
   [ModelTask.FACIAL_RECOGNITION]: {
     [ModelType.DETECTION]: ModelOptions & { options: { minScore: number } };
@@ -53,7 +74,7 @@ export interface Face {
 
 export type FacialRecognitionResponse = { [ModelTask.FACIAL_RECOGNITION]: Face[] } & VisualResponse;
 export type DetectedFaces = { faces: Face[] } & VisualResponse;
-export type MachineLearningRequest = ClipVisualRequest | ClipTextualRequest | FacialRecognitionRequest;
+export type MachineLearningRequest = ClipVisualRequest | ClipTextualRequest | FacialRecognitionRequest | OcrRequest;
 export type TextEncodingOptions = ModelOptions & { language?: string };
 
 @Injectable()
@@ -197,6 +218,17 @@ export class MachineLearningRepository {
     return response[ModelTask.SEARCH];
   }
 
+  async ocr(imagePath: string, { modelName, minDetectionScore, minRecognitionScore, maxResolution }: OcrOptions) {
+    const request = {
+      [ModelTask.OCR]: {
+        [ModelType.DETECTION]: { modelName, options: { minScore: minDetectionScore, maxResolution } },
+        [ModelType.RECOGNITION]: { modelName, options: { minScore: minRecognitionScore } },
+      },
+    };
+    const response = await this.predict<OcrResponse>({ imagePath }, request);
+    return response[ModelTask.OCR];
+  }
+
   private async getFormData(payload: ModelPayload, config: MachineLearningRequest): Promise<FormData> {
     const formData = new FormData();
     formData.append('entries', JSON.stringify(config));

server/src/repositories/ocr.repository.ts

@@ -0,0 +1,68 @@
+import { Injectable } from '@nestjs/common';
+import { Insertable, Kysely, sql } from 'kysely';
+import { InjectKysely } from 'nestjs-kysely';
+import { DummyValue, GenerateSql } from 'src/decorators';
+import { DB } from 'src/schema';
+import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table';
+
+@Injectable()
+export class OcrRepository {
+  constructor(@InjectKysely() private db: Kysely<DB>) {}
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getById(id: string) {
+    return this.db.selectFrom('asset_ocr').selectAll('asset_ocr').where('asset_ocr.id', '=', id).executeTakeFirst();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getByAssetId(id: string) {
+    return this.db.selectFrom('asset_ocr').selectAll('asset_ocr').where('asset_ocr.assetId', '=', id).execute();
+  }
+
+  deleteAll() {
+    return this.db.transaction().execute(async (trx: Kysely<DB>) => {
+      await sql`truncate ${sql.table('asset_ocr')}`.execute(trx);
+      await sql`truncate ${sql.table('ocr_search')}`.execute(trx);
+    });
+  }
+
+  @GenerateSql({
+    params: [
+      DummyValue.UUID,
+      [
+        {
+          assetId: DummyValue.UUID,
+          x1: DummyValue.NUMBER,
+          y1: DummyValue.NUMBER,
+          x2: DummyValue.NUMBER,
+          y2: DummyValue.NUMBER,
+          x3: DummyValue.NUMBER,
+          y3: DummyValue.NUMBER,
+          x4: DummyValue.NUMBER,
+          y4: DummyValue.NUMBER,
+          text: DummyValue.STRING,
+          boxScore: DummyValue.NUMBER,
+          textScore: DummyValue.NUMBER,
+        },
+      ],
+    ],
+  })
+  upsert(assetId: string, ocrDataList: Insertable<AssetOcrTable>[]) {
+    let query = this.db.with('deleted_ocr', (db) => db.deleteFrom('asset_ocr').where('assetId', '=', assetId));
+    if (ocrDataList.length > 0) {
+      const searchText = ocrDataList.map((item) => item.text.trim()).join(' ');
+      (query as any) = query
+        .with('inserted_ocr', (db) => db.insertInto('asset_ocr').values(ocrDataList))
+        .with('inserted_search', (db) =>
+          db
+            .insertInto('ocr_search')
+            .values({ assetId, text: searchText })
+            .onConflict((oc) => oc.column('assetId').doUpdateSet((eb) => ({ text: eb.ref('excluded.text') }))),
+        );
+    } else {
+      (query as any) = query.with('deleted_search', (db) => db.deleteFrom('ocr_search').where('assetId', '=', assetId));
+    }
+
+    return query.selectNoFrom(sql`1`.as('dummy')).execute();
+  }
+}

server/src/repositories/search.repository.ts

@@ -84,6 +84,10 @@ export interface SearchEmbeddingOptions {
   userIds: string[];
 }
 
+export interface SearchOcrOptions {
+  ocr?: string;
+}
+
 export interface SearchPeopleOptions {
   personIds?: string[];
 }
@@ -114,7 +118,8 @@ type BaseAssetSearchOptions = SearchDateOptions &
   SearchUserIdOptions &
   SearchPeopleOptions &
   SearchTagOptions &
-  SearchAlbumOptions;
+  SearchAlbumOptions &
+  SearchOcrOptions;
 
 export type AssetSearchOptions = BaseAssetSearchOptions & SearchRelationOptions;
 
@@ -127,7 +132,10 @@ export type SmartSearchOptions = SearchDateOptions &
   SearchStatusOptions &
   SearchUserIdOptions &
   SearchPeopleOptions &
-  SearchTagOptions;
+  SearchTagOptions &
+  SearchOcrOptions;
+
+export type OcrSearchOptions = SearchDateOptions & SearchOcrOptions;
 
 export type LargeAssetSearchOptions = AssetSearchOptions & { minFileSize?: number };