diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index 26f04bd9cb..63174f0b0f 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -132,6 +132,14 @@ where "assetId" = "asset"."id" and "asset_file"."type" = $3 ) + or not exists ( + select + from + "asset_file" + where + "assetId" = "asset"."id" + and "asset_file"."type" = $4 + ) or "asset"."thumbhash" is null ) diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 6fec1b38a7..4c6e665c4b 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -57,8 +57,8 @@ export class AssetJobRepository { .executeTakeFirst(); } - @GenerateSql({ params: [false], stream: true }) - streamForThumbnailJob(force: boolean) { + @GenerateSql({ params: [{ force: false, fullsizeEnabled: true }], stream: true }) + streamForThumbnailJob(options: { force: boolean | undefined; fullsizeEnabled: boolean }) { return this.db .selectFrom('asset') .select(['asset.id', 'asset.thumbhash']) @@ -66,12 +66,12 @@ export class AssetJobRepository { .select(withEdits) .where('asset.deletedAt', 'is', null) .where('asset.visibility', '!=', AssetVisibility.Hidden) - .$if(!force, (qb) => + .$if(!options.force, (qb) => qb // If there aren't any entries, metadata extraction hasn't run yet which is required for thumbnails .innerJoin('asset_job_status', 'asset_job_status.assetId', 'asset.id') - .where((eb) => - eb.or([ + .where((eb) => { + const conditions = [ eb.not((eb) => eb.exists((qb) => qb @@ -88,9 +88,25 @@ export class AssetJobRepository { .where('asset_file.type', '=', AssetFileType.Thumbnail), ), ), - eb('asset.thumbhash', 'is', null), - ]), - ), + ]; + + if (options.fullsizeEnabled) { + conditions.push( + eb.not((eb) => + eb.exists((qb) => + qb + .selectFrom('asset_file') + .whereRef('assetId', '=', 'asset.id') + .where('asset_file.type', '=', AssetFileType.FullSize), + ), + ), + ); + } + + conditions.push(eb('asset.thumbhash', 'is', null)); + + return eb.or(conditions); + }), ) .stream(); } diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index fa2607faa9..75812e2fcb 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -23,8 +23,14 @@ import { faceStub } from 'test/fixtures/face.stub'; import { probeStub } from 'test/fixtures/media.stub'; import { personStub, personThumbnailStub } from 'test/fixtures/person.stub'; import { systemConfigStub } from 'test/fixtures/system-config.stub'; +import { factory } from 'test/small.factory'; import { makeStream, newTestService, ServiceMocks } from 'test/utils'; +const filesNoFullsize = [ + factory.assetFile({ type: AssetFileType.Preview }), + factory.assetFile({ type: AssetFileType.Thumbnail }), +]; + const fullsizeBuffer = Buffer.from('embedded image data'); const rawBuffer = Buffer.from('raw image data'); const extractedBuffer = Buffer.from('embedded image file'); @@ -49,7 +55,7 @@ describe(MediaService.name, () => { await sut.handleQueueGenerateThumbnails({ force: true }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, @@ -72,7 +78,7 @@ describe(MediaService.name, () => { await sut.handleQueueGenerateThumbnails({ force: true }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, @@ -87,7 +93,7 @@ describe(MediaService.name, () => { await sut.handleQueueGenerateThumbnails({ force: true }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, @@ -103,7 +109,7 @@ describe(MediaService.name, () => { await sut.handleQueueGenerateThumbnails({ force: false }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' }); expect(mocks.person.getRandomFace).toHaveBeenCalled(); expect(mocks.person.update).toHaveBeenCalledTimes(1); @@ -122,7 +128,7 @@ describe(MediaService.name, () => { mocks.person.getAll.mockReturnValue(makeStream()); await sut.handleQueueGenerateThumbnails({ force: false }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, @@ -138,7 +144,7 @@ describe(MediaService.name, () => { mocks.person.getAll.mockReturnValue(makeStream()); await sut.handleQueueGenerateThumbnails({ force: false }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, @@ -154,7 +160,7 @@ describe(MediaService.name, () => { mocks.person.getAll.mockReturnValue(makeStream()); await sut.handleQueueGenerateThumbnails({ force: false }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, @@ -165,12 +171,43 @@ describe(MediaService.name, () => { expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' }); }); + it('should queue all assets with missing fullsize when feature is enabled', async () => { + mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } }); + const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize }; + mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset])); + mocks.person.getAll.mockReturnValue(makeStream()); + await sut.handleQueueGenerateThumbnails({ force: false }); + + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: true }); + expect(mocks.job.queueAll).toHaveBeenCalledWith([ + { + name: JobName.AssetGenerateThumbnails, + data: { id: asset.id }, + }, + ]); + + expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' }); + }); + + it('should not queue assets with missing fullsize when feature is disabled', async () => { + mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } }); + const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize }; + mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset])); + mocks.person.getAll.mockReturnValue(makeStream()); + await sut.handleQueueGenerateThumbnails({ force: false }); + + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); + expect(mocks.job.queueAll).toHaveBeenCalledWith([]); + + expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' }); + }); + it('should queue assets with edits but missing edited thumbnails', async () => { mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit])); mocks.person.getAll.mockReturnValue(makeStream()); await sut.handleQueueGenerateThumbnails({ force: false }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetEditThumbnailGeneration, @@ -181,12 +218,42 @@ describe(MediaService.name, () => { expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' }); }); + it('should not queue assets with missing edited fullsize when feature is disabled', async () => { + mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } }); + mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit])); + mocks.person.getAll.mockReturnValue(makeStream()); + await sut.handleQueueGenerateThumbnails({ force: false }); + + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false }); + expect(mocks.job.queueAll).toHaveBeenCalledWith([]); + + expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' }); + }); + + it('should queue assets with missing fullsize when force is true, regardless of setting', async () => { + mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } }); + const asset = { id: factory.uuid(), thumbhash: Buffer.from('thumbhash'), edits: [], files: filesNoFullsize }; + mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset])); + mocks.person.getAll.mockReturnValue(makeStream()); + await sut.handleQueueGenerateThumbnails({ force: true }); + + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false }); + expect(mocks.job.queueAll).toHaveBeenCalledWith([ + { + name: JobName.AssetGenerateThumbnails, + data: { id: asset.id }, + }, + ]); + + expect(mocks.person.getAll).toHaveBeenCalled(); + }); + it('should queue both regular and edited thumbnails for assets with edits when force is true', async () => { mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit])); mocks.person.getAll.mockReturnValue(makeStream()); await sut.handleQueueGenerateThumbnails({ force: true }); - expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true); + expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false }); expect(mocks.job.queueAll).toHaveBeenCalledWith([ { name: JobName.AssetGenerateThumbnails, diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index b9b8d74737..00bd0305dd 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -68,6 +68,7 @@ export class MediaService extends BaseService { @OnJob({ name: JobName.AssetGenerateThumbnailsQueueAll, queue: QueueName.ThumbnailGeneration }) async handleQueueGenerateThumbnails({ force }: JobOf): Promise { + const config = await this.getConfig({ withCache: true }); let jobs: JobItem[] = []; const queueAll = async () => { @@ -75,16 +76,18 @@ export class MediaService extends BaseService { jobs = []; }; - for await (const asset of this.assetJobRepository.streamForThumbnailJob(!!force)) { - const assetFiles = getAssetFiles(asset.files); + const fullsizeEnabled = config.image.fullsize.enabled; + for await (const asset of this.assetJobRepository.streamForThumbnailJob({ force, fullsizeEnabled })) { + const { previewFile, thumbnailFile, fullsizeFile, editedPreviewFile, editedThumbnailFile, editedFullsizeFile } = + getAssetFiles(asset.files); - if (!assetFiles.previewFile || !assetFiles.thumbnailFile || !asset.thumbhash || force) { + if (force || !previewFile || !thumbnailFile || !asset.thumbhash || (fullsizeEnabled && !fullsizeFile)) { jobs.push({ name: JobName.AssetGenerateThumbnails, data: { id: asset.id } }); } if ( asset.edits.length > 0 && - (!assetFiles.editedPreviewFile || !assetFiles.editedThumbnailFile || !assetFiles.editedFullsizeFile || force) + (force || !editedPreviewFile || !editedThumbnailFile || (fullsizeEnabled && !editedFullsizeFile)) ) { jobs.push({ name: JobName.AssetEditThumbnailGeneration, data: { id: asset.id } }); } diff --git a/server/test/small.factory.ts b/server/test/small.factory.ts index 6ce4ad1f59..579da1a2d8 100644 --- a/server/test/small.factory.ts +++ b/server/test/small.factory.ts @@ -533,6 +533,7 @@ export const factory = { assetEdit: assetEditFactory, tag: tagFactory, uuid: newUuid, + buffer: () => Buffer.from('this is a fake buffer'), date: newDate, responses: { badRequest: (message: any = null) => ({