github-mirrors/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-03-11 12:47:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): pagination for asset queries in jobs (#2516)

Browse Source 
* feat(server): pagination for asset queries in jobs

* default mock value for getAll

* remove live photo name correction

* order paginated results by createdAt

* change log level

* move usePagination to domain
This commit is contained in:
Michel Heusschen
2023-05-22 20:05:06 +02:00
committed by GitHub
parent feadc45e75
commit f1384fea58
18 changed files with 253 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
server/libs/domain/src/media/media.service.spec.ts
View File

@@ -44,7 +44,10 @@ describe(MediaService.name, () => {
describe('handleQueueGenerateThumbnails', () => {
it('should queue all assets', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
assetMock.getAll.mockResolvedValue({
items: [assetEntityStub.image],
hasNextPage: false,
});
await sut.handleQueueGenerateThumbnails({ force: true });
@@ -57,12 +60,15 @@ describe(MediaService.name, () => {
});
it('should queue all assets with missing thumbnails', async () => {
assetMock.getWithout.mockResolvedValue([assetEntityStub.image]);
assetMock.getWithout.mockResolvedValue({
items: [assetEntityStub.image],
hasNextPage: false,
});
await sut.handleQueueGenerateThumbnails({ force: false });
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.THUMBNAIL);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_JPEG_THUMBNAIL,
data: { asset: assetEntityStub.image },
@@ -183,11 +189,14 @@ describe(MediaService.name, () => {
describe('handleQueueVideoConversion', () => {
it('should queue all video assets', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.video]);
assetMock.getAll.mockResolvedValue({
items: [assetEntityStub.video],
hasNextPage: false,
});
await sut.handleQueueVideoConversion({ force: true });
expect(assetMock.getAll).toHaveBeenCalledWith({ type: AssetType.VIDEO });
expect(assetMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { type: AssetType.VIDEO });
expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.VIDEO_CONVERSION,
@@ -196,12 +205,15 @@ describe(MediaService.name, () => {
});
it('should queue all video assets without encoded videos', async () => {
assetMock.getWithout.mockResolvedValue([assetEntityStub.video]);
assetMock.getWithout.mockResolvedValue({
items: [assetEntityStub.video],
hasNextPage: false,
});
await sut.handleQueueVideoConversion({});
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.ENCODED_VIDEO);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.ENCODED_VIDEO);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.VIDEO_CONVERSION,
data: { asset: assetEntityStub.video },

32
server/libs/domain/src/media/media.service.ts
View File

@@ -3,7 +3,8 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
import { join } from 'path';
import { IAssetRepository, mapAsset, WithoutProperty } from '../asset';
import { CommunicationEvent, ICommunicationRepository } from '../communication';
import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job';
import { usePagination } from '../domain.util';
import { IAssetJob, IBaseJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
@@ -31,12 +32,16 @@ export class MediaService {
try {
const { force } = job;
const assets = force
? await this.assetRepository.getAll()
: await this.assetRepository.getWithout(WithoutProperty.THUMBNAIL);
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
: this.assetRepository.getWithout(pagination, WithoutProperty.THUMBNAIL);
});
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}
}
} catch (error: any) {
this.logger.error('Failed to queue generate thumbnail jobs', error.stack);
@@ -115,11 +120,16 @@ export class MediaService {
const { force } = job;
try {
const assets = force
? await this.assetRepository.getAll({ type: AssetType.VIDEO })
: await this.assetRepository.getWithout(WithoutProperty.ENCODED_VIDEO);
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination, { type: AssetType.VIDEO })
: this.assetRepository.getWithout(pagination, WithoutProperty.ENCODED_VIDEO);
});
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
}
} catch (error: any) {
this.logger.error('Failed to queue video conversions', error.stack);