github-mirrors/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-03-04 09:57:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): library syncing (#12220)

Browse Source 
* refactor: library scanning

fix tests

remove offline files step

cleanup library service

improve tests

cleanup tests

add db migration

fix e2e

cleanup openapi

fix tests

fix tests

update docs

update docs

update mobile code

fix formatting

don't remove assets from library with invalid import path

use trash for offline files

add migration

simplify scan endpoint

cleanup library panel

fix library tests

e2e lint

fix e2e

trash e2e

fix lint

add asset trash tests

add more tests

ensure thumbs are generated

cleanup svelte

cleanup queue names

fix tests

fix lint

add warning due to trash

fix trash tests

fix lint

fix tests

Admin message for offline asset

fix comments

Update web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

add permission to library scan endpoint

revert asset interface sort

add trash reason to shared link stub

improve path view in offline

update docs

improve trash performance

fix comments

remove stray comment

* refactor: add back isOffline and remove trashReason from asset, change sync job flow

* chore(server): drop coverage to 80% for functions

* chore: rebase and generated files

---------

Co-authored-by: Zack Pollard <zackpollard@ymail.com>
This commit is contained in:
Jonathan Jogenfors
2024-09-25 19:26:19 +02:00
committed by GitHub
parent 1ef2834603
commit b2f2be3485
85 changed files with 941 additions and 1926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
server/src/repositories/asset.repository.ts
View File

@@ -13,7 +13,6 @@ import {
AssetDeltaSyncOptions,
AssetExploreFieldOptions,
AssetFullSyncOptions,
AssetPathEntity,
AssetStats,
AssetStatsOptions,
AssetUpdateAllOptions,
@@ -177,14 +176,6 @@ export class AssetRepository implements IAssetRepository {
return this.getAll(pagination, { ...options, userIds: [userId] });
}
@GenerateSql({ params: [{ take: 1, skip: 0 }, DummyValue.UUID] })
getExternalLibraryAssetPaths(pagination: PaginationOptions, libraryId: string): Paginated<AssetPathEntity> {
return paginate(this.repository, pagination, {
select: { id: true, originalPath: true, isOffline: true },
where: { library: { id: libraryId }, isExternal: true },
});
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] })
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null> {
return this.repository.findOne({
@@ -198,24 +189,16 @@ export class AssetRepository implements IAssetRepository {
async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]> {
const result = await this.repository.query(
`
WITH paths AS (SELECT unnest($2::text[]) AS path)
SELECT path FROM paths
WHERE NOT EXISTS (SELECT 1 FROM assets WHERE "libraryId" = $1 AND "originalPath" = path);
`,
WITH paths AS (SELECT unnest($2::text[]) AS path)
SELECT path
FROM paths
WHERE NOT EXISTS (SELECT 1 FROM assets WHERE "libraryId" = $1 AND "originalPath" = path);
`,
[libraryId, originalPaths],
);
return result.map((row: { path: string }) => row.path);
}
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
@ChunkedArray({ paramIndex: 1 })
async updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void> {
await this.repository.update(
{ library: { id: libraryId }, originalPath: Not(In(originalPaths)), isOffline: false },
{ isOffline: true },
);
}
getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
let builder = this.repository.createQueryBuilder('asset').leftJoinAndSelect('asset.files', 'files');
builder = searchAssetBuilder(builder, options);
@@ -373,12 +356,10 @@ export class AssetRepository implements IAssetRepository {
}
@GenerateSql(
...Object.values(WithProperty)
.filter((property) => property !== WithProperty.IS_OFFLINE && property !== WithProperty.IS_ONLINE)
.map((property) => ({
name: property,
params: [DummyValue.PAGINATION, property],
})),
...Object.values(WithProperty).map((property) => ({
name: property,
params: [DummyValue.PAGINATION, property],
})),
)
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity> {
let relations: FindOptionsRelations<AssetEntity> = {};
@@ -531,26 +512,16 @@ export class AssetRepository implements IAssetRepository {
where = [{ sidecarPath: Not(IsNull()), isVisible: true }];
break;
}
case WithProperty.IS_OFFLINE: {
if (!libraryId) {
throw new Error('Library id is required when finding offline assets');
}
where = [{ isOffline: true, libraryId }];
break;
}
case WithProperty.IS_ONLINE: {
if (!libraryId) {
throw new Error('Library id is required when finding online assets');
}
where = [{ isOffline: false, libraryId }];
break;
}
default: {
throw new Error(`Invalid getWith property: ${property}`);
}
}
if (libraryId) {
where = [{ ...where, libraryId }];
}
return paginate(this.repository, pagination, {
where,
withDeleted,
@@ -750,7 +721,10 @@ export class AssetRepository implements IAssetRepository {
builder.andWhere(`asset.deletedAt ${options.isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted();
if (options.isTrashed) {
builder.andWhere('asset.status = :status', { status: AssetStatus.TRASHED });
// TODO: Temporarily inverted to support showing offline assets in the trash queries.
// Once offline assets are handled in a separate screen, this should be set back to status = TRASHED
// and the offline screens should use a separate isOffline = true parameter in the timeline query.
builder.andWhere('asset.status != :status', { status: AssetStatus.DELETED });
}
}

10
server/src/repositories/job.repository.ts
View File

@@ -79,12 +79,12 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.SIDECAR_WRITE]: QueueName.SIDECAR,
// Library management
[JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY,
[JobName.LIBRARY_SCAN]: QueueName.LIBRARY,
[JobName.LIBRARY_SYNC_FILE]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_SYNC_FILES]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_SYNC_ASSETS]: QueueName.LIBRARY,
[JobName.LIBRARY_DELETE]: QueueName.LIBRARY,
[JobName.LIBRARY_CHECK_OFFLINE]: QueueName.LIBRARY,
[JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY,
[JobName.LIBRARY_SYNC_ASSET]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_SYNC_ALL]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY,
// Notification

11
server/src/repositories/trash.repository.ts
View File

@@ -3,7 +3,7 @@ import { AssetEntity } from 'src/entities/asset.entity';
import { AssetStatus } from 'src/enum';
import { ITrashRepository } from 'src/interfaces/trash.interface';
import { Paginated, paginatedBuilder, PaginationOptions } from 'src/utils/pagination';
import { In, IsNull, Not, Repository } from 'typeorm';
import { In, Repository } from 'typeorm';
export class TrashRepository implements ITrashRepository {
constructor(@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>) {}
@@ -26,7 +26,7 @@ export class TrashRepository implements ITrashRepository {
async restore(userId: string): Promise<number> {
const result = await this.assetRepository.update(
{ ownerId: userId, deletedAt: Not(IsNull()) },
{ ownerId: userId, status: AssetStatus.TRASHED },
{ status: AssetStatus.ACTIVE, deletedAt: null },
);
@@ -35,7 +35,7 @@ export class TrashRepository implements ITrashRepository {
async empty(userId: string): Promise<number> {
const result = await this.assetRepository.update(
{ ownerId: userId, deletedAt: Not(IsNull()), status: AssetStatus.TRASHED },
{ ownerId: userId, status: AssetStatus.TRASHED },
{ status: AssetStatus.DELETED },
);
@@ -43,7 +43,10 @@ export class TrashRepository implements ITrashRepository {
}
async restoreAll(ids: string[]): Promise<number> {
const result = await this.assetRepository.update({ id: In(ids) }, { status: AssetStatus.ACTIVE, deletedAt: null });
const result = await this.assetRepository.update(
{ id: In(ids), status: AssetStatus.TRASHED },
{ status: AssetStatus.ACTIVE, deletedAt: null },
);
return result.affected ?? 0;
}
}