diff --git a/server/src/database.ts b/server/src/database.ts index ec614df9e0..caf80aede5 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -5,6 +5,7 @@ import { AssetFileType, AssetType, AssetVisibility, + ChecksumAlgorithm, MemoryType, Permission, PluginContext, @@ -111,6 +112,7 @@ export type Memory = { export type Asset = { id: string; checksum: Buffer; + checksumAlgorithm: ChecksumAlgorithm; deviceAssetId: string; deviceId: string; fileCreatedAt: Date; @@ -329,6 +331,7 @@ export const columns = { asset: [ 'asset.id', 'asset.checksum', + 'asset.checksumAlgorithm', 'asset.deviceAssetId', 'asset.deviceId', 'asset.fileCreatedAt', diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index a76df4abaa..75d9dee79a 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -13,7 +13,7 @@ import { } from 'src/dtos/person.dto'; import { TagResponseDto, mapTag } from 'src/dtos/tag.dto'; import { UserResponseDto, mapUser } from 'src/dtos/user.dto'; -import { AssetStatus, AssetType, AssetVisibility } from 'src/enum'; +import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum'; import { ImageDimensions } from 'src/types'; import { getDimensions } from 'src/utils/asset.util'; import { hexOrBufferToBase64 } from 'src/utils/bytes'; @@ -147,6 +147,7 @@ export type MapAsset = { updateId: string; status: AssetStatus; checksum: Buffer; + checksumAlgorithm: ChecksumAlgorithm; deviceAssetId: string; deviceId: string; duplicateId: string | null; diff --git a/server/src/enum.ts b/server/src/enum.ts index 2aa9bd2aa6..7aa3b50c42 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -37,6 +37,11 @@ export enum AssetType { Other = 'OTHER', } +export enum ChecksumAlgorithm { + sha1File = 'sha1-file', // sha1 checksum of the whole file contents + sha1Path = 'sha1-path', // sha1 checksum of "path:" plus the file path, currently used in external libraries, deprecated +} + export enum AssetFileType { /** * An full/large-size image extracted/converted from RAW photos diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index a9c407782b..0c0ea69a48 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -250,6 +250,7 @@ where select "asset"."id", "asset"."checksum", + "asset"."checksumAlgorithm", "asset"."deviceAssetId", "asset"."deviceId", "asset"."fileCreatedAt", diff --git a/server/src/schema/enums.ts b/server/src/schema/enums.ts index c68f152779..f63a09c462 100644 --- a/server/src/schema/enums.ts +++ b/server/src/schema/enums.ts @@ -1,5 +1,5 @@ import { registerEnum } from '@immich/sql-tools'; -import { AssetStatus, AssetVisibility, SourceType } from 'src/enum'; +import { AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType } from 'src/enum'; export const assets_status_enum = registerEnum({ name: 'assets_status_enum', @@ -15,3 +15,8 @@ export const asset_visibility_enum = registerEnum({ name: 'asset_visibility_enum', values: Object.values(AssetVisibility), }); + +export const asset_checksum_algorithm_enum = registerEnum({ + name: 'asset_checksum_algorithm_enum', + values: Object.values(ChecksumAlgorithm), +}); diff --git a/server/src/schema/migrations/1772202147901-AddChecksumAlgorithm.ts.ts b/server/src/schema/migrations/1772202147901-AddChecksumAlgorithm.ts.ts new file mode 100644 index 0000000000..f384e2b394 --- /dev/null +++ b/server/src/schema/migrations/1772202147901-AddChecksumAlgorithm.ts.ts @@ -0,0 +1,35 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql`CREATE TYPE "asset_checksum_algorithm_enum" AS ENUM ('sha1-file','sha1-path');`.execute(db); + await sql`ALTER TABLE "asset" ADD "checksumAlgorithm" asset_checksum_algorithm_enum;`.execute(db); + + // Update in batches to handle millions of rows efficiently + const batchSize = 10_000; + let updatedRows: number; + + do { + const result = await sql` + UPDATE "asset" + SET "checksumAlgorithm" = CASE + WHEN "isExternal" = true THEN 'sha1-path'::asset_checksum_algorithm_enum + ELSE 'sha1-file'::asset_checksum_algorithm_enum + END + WHERE "id" IN ( + SELECT "id" + FROM "asset" + WHERE "checksumAlgorithm" IS NULL + LIMIT ${batchSize} + ) + `.execute(db); + + updatedRows = Number(result.numAffectedRows ?? 0); + } while (updatedRows > 0); + + await sql`ALTER TABLE "asset" ALTER COLUMN "checksumAlgorithm" SET NOT NULL;`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`ALTER TABLE "asset" DROP COLUMN "checksumAlgorithm";`.execute(db); + await sql`DROP TYPE "asset_checksum_algorithm_enum";`.execute(db); +} \ No newline at end of file diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index 12e9c36125..7abf76ce4b 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -12,8 +12,8 @@ import { UpdateDateColumn, } from '@immich/sql-tools'; import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; -import { AssetStatus, AssetType, AssetVisibility } from 'src/enum'; -import { asset_visibility_enum, assets_status_enum } from 'src/schema/enums'; +import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum'; +import { asset_checksum_algorithm_enum, asset_visibility_enum, assets_status_enum } from 'src/schema/enums'; import { asset_delete_audit } from 'src/schema/functions'; import { LibraryTable } from 'src/schema/tables/library.table'; import { StackTable } from 'src/schema/tables/stack.table'; @@ -98,6 +98,9 @@ export class AssetTable { @Column({ type: 'bytea', index: true }) checksum!: Buffer; // sha1 checksum + @Column({ enum: asset_checksum_algorithm_enum }) + checksumAlgorithm!: ChecksumAlgorithm; + @ForeignKeyColumn(() => AssetTable, { nullable: true, onUpdate: 'CASCADE', onDelete: 'SET NULL' }) livePhotoVideoId!: string | null; diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 020bda4df7..6f43892f10 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -27,6 +27,7 @@ import { AssetStatus, AssetVisibility, CacheControl, + ChecksumAlgorithm, JobName, Permission, StorageFolder, @@ -409,6 +410,7 @@ export class AssetMediaService extends BaseService { deviceId: asset.deviceId, type: asset.type, checksum: asset.checksum, + checksumAlgorithm: asset.checksumAlgorithm, fileCreatedAt: asset.fileCreatedAt, localDateTime: asset.localDateTime, fileModifiedAt: asset.fileModifiedAt, @@ -430,6 +432,7 @@ export class AssetMediaService extends BaseService { libraryId: null, checksum: file.checksum, + checksumAlgorithm: ChecksumAlgorithm.sha1File, originalPath: file.originalPath, deviceAssetId: dto.deviceAssetId, diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 841fa4743c..9f2d69bab5 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -17,7 +17,17 @@ import { ValidateLibraryImportPathResponseDto, ValidateLibraryResponseDto, } from 'src/dtos/library.dto'; -import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; +import { + AssetStatus, + AssetType, + ChecksumAlgorithm, + CronJob, + DatabaseLock, + ImmichWorker, + JobName, + JobStatus, + QueueName, +} from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { AssetSyncResult } from 'src/repositories/library.repository'; import { AssetTable } from 'src/schema/tables/asset.table'; @@ -400,6 +410,7 @@ export class LibraryService extends BaseService { ownerId, libraryId, checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`), + checksumAlgorithm: ChecksumAlgorithm.sha1Path, originalPath: assetPath, fileCreatedAt: stat.mtime, diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index 92ec13bea5..2fb9d6d7a9 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -7,6 +7,7 @@ import { AssetFileType, AssetType, AssetVisibility, + ChecksumAlgorithm, ExifOrientation, ImmichWorker, JobName, @@ -651,6 +652,7 @@ describe(MetadataService.name, () => { expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id); expect(mocks.asset.create).toHaveBeenCalledWith({ checksum: expect.any(Buffer), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: 'NONE', deviceId: 'NONE', fileCreatedAt: asset.fileCreatedAt, @@ -704,6 +706,7 @@ describe(MetadataService.name, () => { expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id); expect(mocks.asset.create).toHaveBeenCalledWith({ checksum: expect.any(Buffer), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: 'NONE', deviceId: 'NONE', fileCreatedAt: asset.fileCreatedAt, @@ -757,6 +760,7 @@ describe(MetadataService.name, () => { expect(mocks.storage.readFile).toHaveBeenCalledWith(asset.originalPath, expect.any(Object)); expect(mocks.asset.create).toHaveBeenCalledWith({ checksum: expect.any(Buffer), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: 'NONE', deviceId: 'NONE', fileCreatedAt: asset.fileCreatedAt, diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index f22d4682fa..03348ae96e 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -14,6 +14,7 @@ import { AssetFileType, AssetType, AssetVisibility, + ChecksumAlgorithm, DatabaseLock, ExifOrientation, ImmichWorker, @@ -675,6 +676,7 @@ export class MetadataService extends BaseService { fileModifiedAt: stats.mtime, localDateTime: dates.localDateTime, checksum, + checksumAlgorithm: ChecksumAlgorithm.sha1File, ownerId: asset.ownerId, originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId), originalFileName: `${parse(asset.originalFileName).name}.mp4`, diff --git a/server/test/factories/asset.factory.ts b/server/test/factories/asset.factory.ts index 4d54ba820b..f48ab53da5 100644 --- a/server/test/factories/asset.factory.ts +++ b/server/test/factories/asset.factory.ts @@ -1,5 +1,5 @@ import { Selectable } from 'kysely'; -import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum'; +import { AssetFileType, AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum'; import { AssetTable } from 'src/schema/tables/asset.table'; import { StackTable } from 'src/schema/tables/stack.table'; import { AssetEditFactory } from 'test/factories/asset-edit.factory'; @@ -51,6 +51,7 @@ export class AssetFactory { updateId: newUuidV7(), status: AssetStatus.Active, checksum: newSha1(), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: '', deviceId: '', duplicateId: null, diff --git a/server/test/fixtures/shared-link.stub.ts b/server/test/fixtures/shared-link.stub.ts index a42ff743bc..7681f0e8b2 100644 --- a/server/test/fixtures/shared-link.stub.ts +++ b/server/test/fixtures/shared-link.stub.ts @@ -1,7 +1,7 @@ import { UserAdmin } from 'src/database'; import { MapAsset } from 'src/dtos/asset-response.dto'; import { SharedLinkResponseDto } from 'src/dtos/shared-link.dto'; -import { AssetStatus, AssetType, AssetVisibility, SharedLinkType } from 'src/enum'; +import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm, SharedLinkType } from 'src/enum'; import { AssetFactory } from 'test/factories/asset.factory'; import { authStub } from 'test/fixtures/auth.stub'; import { userStub } from 'test/fixtures/user.stub'; @@ -94,6 +94,7 @@ export const sharedLinkStub = { type: AssetType.Video, originalPath: 'fake_path/jpeg', checksum: Buffer.from('file hash', 'utf8'), + checksumAlgorithm: ChecksumAlgorithm.sha1File, fileModifiedAt: today, fileCreatedAt: today, localDateTime: today, diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index 51dde6b36b..5ffa2d7312 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -11,6 +11,7 @@ import { AlbumUserRole, AssetType, AssetVisibility, + ChecksumAlgorithm, MemoryType, SourceType, SyncEntityType, @@ -535,6 +536,7 @@ const assetInsert = (asset: Partial> = {}) => { deviceId: '', originalFileName: '', checksum: randomBytes(32), + checksumAlgorithm: ChecksumAlgorithm.sha1File, type: AssetType.Image, originalPath: '/path/to/something.jpg', ownerId: 'not-a-valid-uuid', diff --git a/server/test/small.factory.ts b/server/test/small.factory.ts index 06a5798405..c15ecb3ac2 100644 --- a/server/test/small.factory.ts +++ b/server/test/small.factory.ts @@ -28,6 +28,7 @@ import { AssetStatus, AssetType, AssetVisibility, + ChecksumAlgorithm, MemoryType, Permission, SourceType, @@ -249,6 +250,7 @@ const assetFactory = ( updateId: newUuidV7(), status: AssetStatus.Active, checksum: newSha1(), + checksumAlgorithm: ChecksumAlgorithm.sha1File, deviceAssetId: '', deviceId: '', duplicateId: null,