github-mirrors/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-02-14 12:58:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

perf(server): optimize getByIds query (#7918)

Browse Source 
* clean up usage

* i'm not updating all these tests

* update tests

* add indices

* add indices to entities

remove index from person entity

add to face entity

fix

* simplify query

* update sql

* missing await

* remove synchronize false
This commit is contained in:
Mert
2024-03-14 01:58:09 -04:00
committed by GitHub
parent d67cc00e4e
commit ee8e8a0c0f
19 changed files with 120 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/domain/download/download.service.spec.ts
View File

@@ -164,7 +164,7 @@ describe(DownloadService.name, () => {
const assetIds = ['asset-1', 'asset-2'];
await expect(sut.getDownloadInfo(authStub.admin, { assetIds })).resolves.toEqual(downloadResponse);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1', 'asset-2']);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1', 'asset-2'], { exifInfo: true });
});
it('should return a list of archives (albumId)', async () => {
@@ -228,10 +228,10 @@ describe(DownloadService.name, () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(assetIds));
when(assetMock.getByIds)
.calledWith([assetStub.livePhotoStillAsset.id])
.calledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true })
.mockResolvedValue([assetStub.livePhotoStillAsset]);
when(assetMock.getByIds)
.calledWith([assetStub.livePhotoMotionAsset.id])
.calledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true })
.mockResolvedValue([assetStub.livePhotoMotionAsset]);
await expect(sut.getDownloadInfo(authStub.admin, { assetIds })).resolves.toEqual({

4
server/src/domain/download/download.service.ts
View File

@@ -50,7 +50,7 @@ export class DownloadService {
// motion part of live photos
const motionIds = assets.map((asset) => asset.livePhotoVideoId).filter<string>((id): id is string => !!id);
if (motionIds.length > 0) {
assets.push(...(await this.assetRepository.getByIds(motionIds)));
assets.push(...(await this.assetRepository.getByIds(motionIds, { exifInfo: true })));
}
for (const asset of assets) {
@@ -114,7 +114,7 @@ export class DownloadService {
if (dto.assetIds) {
const assetIds = dto.assetIds;
await this.access.requirePermission(auth, Permission.ASSET_DOWNLOAD, assetIds);
const assets = await this.assetRepository.getByIds(assetIds);
const assets = await this.assetRepository.getByIds(assetIds, { exifInfo: true });
return usePagination(PAGINATION_SIZE, () => ({ hasNextPage: false, items: assets }));
}

2
server/src/domain/job/job.service.spec.ts
View File

@@ -330,8 +330,6 @@ describe(JobService.name, () => {
} else {
assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]);
}
} else {
assetMock.getByIds.mockResolvedValue([]);
}
await sut.init(makeMockHandlers(true));

4
server/src/domain/job/job.service.ts
View File

@@ -214,7 +214,7 @@ export class JobService {
case JobName.METADATA_EXTRACTION: {
if (item.data.source === 'sidecar-write') {
const [asset] = await this.assetRepository.getByIds([item.data.id]);
const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]);
if (asset) {
this.communicationRepository.send(ClientEvent.ASSET_UPDATE, asset.ownerId, mapAsset(asset));
}
@@ -272,7 +272,7 @@ export class JobService {
break;
}
const [asset] = await this.assetRepository.getByIds([item.data.id]);
const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]);
// Only live-photo motion part will be marked as not visible immediately on upload. Skip notifying clients
if (asset && asset.isVisible) {

4
server/src/domain/media/media.service.ts
View File

@@ -165,7 +165,7 @@ export class MediaService {
}
async handleGenerateJpegThumbnail({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
if (!asset) {
return false;
}
@@ -215,7 +215,7 @@ export class MediaService {
}
async handleGenerateWebpThumbnail({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
if (!asset) {
return false;
}

10
server/src/domain/metadata/metadata.service.spec.ts
View File

@@ -114,7 +114,7 @@ describe(MetadataService.name, () => {
describe('handleLivePhotoLinking', () => {
it('should handle an asset that could not be found', async () => {
await expect(sut.handleLivePhotoLinking({ id: assetStub.image.id })).resolves.toBe(false);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(assetMock.findLivePhotoMatch).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalled();
expect(albumMock.removeAsset).not.toHaveBeenCalled();
@@ -124,7 +124,7 @@ describe(MetadataService.name, () => {
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, exifInfo: undefined }]);
await expect(sut.handleLivePhotoLinking({ id: assetStub.image.id })).resolves.toBe(false);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(assetMock.findLivePhotoMatch).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalled();
expect(albumMock.removeAsset).not.toHaveBeenCalled();
@@ -134,7 +134,7 @@ describe(MetadataService.name, () => {
assetMock.getByIds.mockResolvedValue([{ ...assetStub.image }]);
await expect(sut.handleLivePhotoLinking({ id: assetStub.image.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(assetMock.findLivePhotoMatch).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalled();
expect(albumMock.removeAsset).not.toHaveBeenCalled();
@@ -149,7 +149,7 @@ describe(MetadataService.name, () => {
]);
await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoMotionAsset.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });
expect(assetMock.findLivePhotoMatch).toHaveBeenCalledWith({
livePhotoCID: assetStub.livePhotoStillAsset.id,
ownerId: assetStub.livePhotoMotionAsset.ownerId,
@@ -170,7 +170,7 @@ describe(MetadataService.name, () => {
assetMock.findLivePhotoMatch.mockResolvedValue(assetStub.livePhotoMotionAsset);
await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });
expect(assetMock.findLivePhotoMatch).toHaveBeenCalledWith({
livePhotoCID: assetStub.livePhotoMotionAsset.id,
ownerId: assetStub.livePhotoStillAsset.ownerId,

2
server/src/domain/metadata/metadata.service.ts
View File

@@ -153,7 +153,7 @@ export class MetadataService {
async handleLivePhotoLinking(job: IEntityJob) {
const { id } = job;
const [asset] = await this.assetRepository.getByIds([id]);
const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
if (!asset?.exifInfo) {
return false;
}

1
server/src/domain/repositories/asset.repository.ts
View File

@@ -121,6 +121,7 @@ export interface IAssetRepository {
relations?: FindOptionsRelations<AssetEntity>,
select?: FindOptionsSelect<AssetEntity>,
): Promise<AssetEntity[]>;
getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
getByDayOfYear(ownerId: string, monthDay: MonthDay): Promise<AssetEntity[]>;
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;

2
server/src/domain/search/search.service.spec.ts
View File

@@ -76,7 +76,7 @@ describe(SearchService.name, () => {
fieldName: 'smartInfo.tags',
items: [{ value: 'train', data: assetStub.imageFrom2015.id }],
});
assetMock.getByIds.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
assetMock.getByIdsWithAllRelations.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
const expectedResponse = [
{ fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] },
{ fieldName: 'smartInfo.tags', items: [{ value: 'train', data: mapAsset(assetStub.imageFrom2015) }] },

2
server/src/domain/search/search.service.ts
View File

@@ -60,7 +60,7 @@ export class SearchService {
this.assetRepository.getAssetIdByTag(auth.user.id, options),
]);
const assetIds = new Set<string>(results.flatMap((field) => field.items.map((item) => item.data)));
const assets = await this.assetRepository.getByIds([...assetIds]);
const assets = await this.assetRepository.getByIdsWithAllRelations([...assetIds]);
const assetMap = new Map<string, AssetResponseDto>(assets.map((asset) => [asset.id, mapAsset(asset)]));
return results.map(({ fieldName, items }) => ({

4
server/src/domain/smart-info/smart-info.service.ts
View File

@@ -76,6 +76,10 @@ export class SmartInfoService {
}
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset) {
return false;
}
if (!asset.resizePath) {
return false;
}

32
server/src/domain/storage-template/storage-template.service.spec.ts
View File

@@ -101,11 +101,11 @@ describe(StorageTemplateService.name, () => {
.mockResolvedValue(assetStub.livePhotoMotionAsset);
when(assetMock.getByIds)
.calledWith([assetStub.livePhotoStillAsset.id])
.calledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true })
.mockResolvedValue([assetStub.livePhotoStillAsset]);
when(assetMock.getByIds)
.calledWith([assetStub.livePhotoMotionAsset.id])
.calledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true })
.mockResolvedValue([assetStub.livePhotoMotionAsset]);
when(moveMock.create)
@@ -140,8 +140,8 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
expect(assetMock.save).toHaveBeenCalledWith({
id: assetStub.livePhotoStillAsset.id,
@@ -172,7 +172,9 @@ describe(StorageTemplateService.name, () => {
.calledWith({ id: assetStub.image.id, originalPath: newPath })
.mockResolvedValue(assetStub.image);
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
when(assetMock.getByIds)
.calledWith([assetStub.image.id], { exifInfo: true })
.mockResolvedValue([assetStub.image]);
when(moveMock.update)
.calledWith({
@@ -190,7 +192,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
expect(moveMock.update).toHaveBeenCalledWith({
@@ -227,7 +229,9 @@ describe(StorageTemplateService.name, () => {
.calledWith({ id: assetStub.image.id, originalPath: newPath })
.mockResolvedValue(assetStub.image);
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
when(assetMock.getByIds)
.calledWith([assetStub.image.id], { exifInfo: true })
.mockResolvedValue([assetStub.image]);
when(moveMock.update)
.calledWith({
@@ -245,7 +249,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
expect(storageMock.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
@@ -275,7 +279,9 @@ describe(StorageTemplateService.name, () => {
.calledWith({ id: assetStub.image.id, originalPath: newPath })
.mockResolvedValue(assetStub.image);
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
when(assetMock.getByIds)
.calledWith([assetStub.image.id], { exifInfo: true })
.mockResolvedValue([assetStub.image]);
when(moveMock.create)
.calledWith({
@@ -294,7 +300,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(1);
expect(storageMock.stat).toHaveBeenCalledWith(newPath);
expect(moveMock.create).toHaveBeenCalledWith({
@@ -340,7 +346,9 @@ describe(StorageTemplateService.name, () => {
.calledWith({ id: assetStub.image.id, originalPath: newPath })
.mockResolvedValue(assetStub.image);
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
when(assetMock.getByIds)
.calledWith([assetStub.image.id], { exifInfo: true })
.mockResolvedValue([assetStub.image]);
when(moveMock.update)
.calledWith({
@@ -358,7 +366,7 @@ describe(StorageTemplateService.name, () => {
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
expect(storageMock.rename).not.toHaveBeenCalled();

10
server/src/domain/storage-template/storage-template.service.ts
View File

@@ -92,7 +92,10 @@ export class StorageTemplateService {
return true;
}
const [asset] = await this.assetRepository.getByIds([id]);
const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
if (!asset) {
return false;
}
const user = await this.userRepository.get(asset.ownerId, {});
const storageLabel = user?.storageLabel || null;
@@ -101,7 +104,10 @@ export class StorageTemplateService {
// move motion part of live photo
if (asset.livePhotoVideoId) {
const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]);
const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId], { exifInfo: true });
if (!livePhotoVideo) {
return false;
}
const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath);
await this.moveAsset(livePhotoVideo, { storageLabel, filename: motionFilename });
}