github-mirrors/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-03-25 19:18:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor: orphan -> untracked

Browse Source
This commit is contained in:
izzy
2026-01-07 12:17:28 +00:00
parent d189722bbf
commit ed33f79e2a
27 changed files with 213 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
server/src/services/integrity.service.spec.ts
View File

@@ -145,7 +145,7 @@ describe(IntegrityService.name, () => {
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAssetId' }]);
});
it('deletes orphaned file', async () => {
it('deletes untracked file', async () => {
mocks.integrityReport.getById.mockResolvedValue({
id: 'id',
createdAt: new Date(0),
@@ -169,7 +169,7 @@ describe(IntegrityService.name, () => {
});
});
describe('handleOrphanedFilesQueueAll', () => {
describe('handleUntrackedFilesQueueAll', () => {
beforeEach(() => {
mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never);
});
@@ -189,11 +189,11 @@ describe(IntegrityService.name, () => {
})() as never,
);
await sut.handleOrphanedFilesQueueAll({ refreshOnly: false });
await sut.handleUntrackedFilesQueueAll({ refreshOnly: false });
expect(mocks.job.queue).toBeCalledTimes(4);
expect(mocks.job.queue).toBeCalledWith({
name: JobName.IntegrityOrphanedFiles,
name: JobName.IntegrityUntrackedFiles,
data: {
type: 'asset',
paths: expect.arrayContaining(['/path/to/file']),
@@ -201,7 +201,7 @@ describe(IntegrityService.name, () => {
});
expect(mocks.job.queue).toBeCalledWith({
name: JobName.IntegrityOrphanedFiles,
name: JobName.IntegrityUntrackedFiles,
data: {
type: 'asset_file',
paths: expect.arrayContaining(['/path/to/file3']),
@@ -216,11 +216,11 @@ describe(IntegrityService.name, () => {
})() as never,
);
await sut.handleOrphanedFilesQueueAll({ refreshOnly: false });
await sut.handleUntrackedFilesQueueAll({ refreshOnly: false });
expect(mocks.job.queue).toBeCalledTimes(1);
expect(mocks.job.queue).toBeCalledWith({
name: JobName.IntegrityOrphanedFilesRefresh,
name: JobName.IntegrityUntrackedFilesRefresh,
data: {
items: expect.arrayContaining(['mockReport']),
},
@@ -228,33 +228,33 @@ describe(IntegrityService.name, () => {
});
it('should succeed', async () => {
await expect(sut.handleOrphanedFilesQueueAll({ refreshOnly: false })).resolves.toBe(JobStatus.Success);
await expect(sut.handleUntrackedFilesQueueAll({ refreshOnly: false })).resolves.toBe(JobStatus.Success);
});
});
describe('handleOrphanedFiles', () => {
it('should detect orphaned asset files', async () => {
describe('handleUntrackedFiles', () => {
it('should detect untracked asset files', async () => {
mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([
{ originalPath: '/path/to/file1', encodedVideoPath: null },
]);
await sut.handleOrphanedFiles({
await sut.handleUntrackedFiles({
type: 'asset',
paths: ['/path/to/file1', '/path/to/orphan'],
paths: ['/path/to/file1', '/path/to/untracked'],
});
expect(mocks.integrityReport.getAssetPathsByPaths).toHaveBeenCalledWith(['/path/to/file1', '/path/to/orphan']);
expect(mocks.integrityReport.getAssetPathsByPaths).toHaveBeenCalledWith(['/path/to/file1', '/path/to/untracked']);
expect(mocks.integrityReport.create).toHaveBeenCalledWith([
{ type: IntegrityReportType.OrphanFile, path: '/path/to/orphan' },
{ type: IntegrityReportType.UntrackedFile, path: '/path/to/untracked' },
]);
});
it('should not create reports when no orphans found for assets', async () => {
it('should not create reports when no untracked files found for assets', async () => {
mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([
{ originalPath: '/path/to/file1', encodedVideoPath: '/path/to/encoded' },
]);
await sut.handleOrphanedFiles({
await sut.handleUntrackedFiles({
type: 'asset',
paths: ['/path/to/file1', '/path/to/encoded'],
});
@@ -262,28 +262,28 @@ describe(IntegrityService.name, () => {
expect(mocks.integrityReport.create).not.toHaveBeenCalled();
});
it('should detect orphaned asset_file files', async () => {
it('should detect untracked asset_file files', async () => {
mocks.integrityReport.getAssetFilePathsByPaths.mockResolvedValue([{ path: '/path/to/thumb1' }]);
await sut.handleOrphanedFiles({
await sut.handleUntrackedFiles({
type: 'asset_file',
paths: ['/path/to/thumb1', '/path/to/orphan_thumb'],
paths: ['/path/to/thumb1', '/path/to/untracked_thumb'],
});
expect(mocks.integrityReport.create).toHaveBeenCalledWith([
{ type: IntegrityReportType.OrphanFile, path: '/path/to/orphan_thumb' },
{ type: IntegrityReportType.UntrackedFile, path: '/path/to/untracked_thumb' },
]);
});
});
describe('handleOrphanedRefresh', () => {
describe('handleUntrackedRefresh', () => {
it('should delete reports for files that no longer exist', async () => {
mocks.storage.stat
.mockRejectedValueOnce(new Error('ENOENT'))
.mockResolvedValueOnce({} as never)
.mockRejectedValueOnce(new Error('ENOENT'));
await sut.handleOrphanedRefresh({
await sut.handleUntrackedRefresh({
items: [
{ reportId: 'report1', path: '/path/to/missing1' },
{ reportId: 'report2', path: '/path/to/existing' },
@@ -297,7 +297,7 @@ describe(IntegrityService.name, () => {
it('should not delete reports for files that still exist', async () => {
mocks.storage.stat.mockResolvedValue({} as never);
await sut.handleOrphanedRefresh({
await sut.handleUntrackedRefresh({
items: [{ reportId: 'report1', path: '/path/to/existing' }],
});
@@ -305,7 +305,7 @@ describe(IntegrityService.name, () => {
});
it('should succeed', async () => {
await expect(sut.handleOrphanedRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
await expect(sut.handleUntrackedRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
});
});
@@ -604,24 +604,24 @@ describe(IntegrityService.name, () => {
expect(mocks.job.queue).toHaveBeenCalledTimes(2);
});
it('should queue delete jobs for orphan file reports', async () => {
it('should queue delete jobs for untracked file reports', async () => {
mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue(
(function* () {
yield { id: 'report1', path: '/path/to/orphan' };
yield { id: 'report1', path: '/path/to/untracked' };
})() as never,
);
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.OrphanFile });
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.UntrackedFile });
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(
undefined,
IntegrityReportType.OrphanFile,
IntegrityReportType.UntrackedFile,
);
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.IntegrityDeleteReports,
data: {
reports: [{ id: 'report1', path: '/path/to/orphan' }],
reports: [{ id: 'report1', path: '/path/to/untracked' }],
},
});
});
@@ -648,7 +648,7 @@ describe(IntegrityService.name, () => {
{ id: 'report1', assetId: 'asset1', fileAssetId: null, path: '/path/to/file1' },
{ id: 'report2', assetId: 'asset2', fileAssetId: null, path: '/path/to/file2' },
{ id: 'report3', assetId: null, fileAssetId: 'fileAsset1', path: '/path/to/file3' },
{ id: 'report4', assetId: null, fileAssetId: null, path: '/path/to/orphan' },
{ id: 'report4', assetId: null, fileAssetId: null, path: '/path/to/untracked' },
],
});
@@ -666,7 +666,7 @@ describe(IntegrityService.name, () => {
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAsset1' }]);
expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/orphan');
expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/untracked');
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report4']);
});

66
server/src/services/integrity.service.ts
View File

@@ -31,15 +31,15 @@ import {
IIntegrityDeleteReportTypeJob,
IIntegrityJob,
IIntegrityMissingFilesJob,
IIntegrityOrphanedFilesJob,
IIntegrityPathWithChecksumJob,
IIntegrityPathWithReportJob,
IIntegrityUntrackedFilesJob,
} from 'src/types';
import { ImmichFileResponse } from 'src/utils/file';
import { handlePromiseError } from 'src/utils/misc';
/**
* Orphan Files:
* Untracked Files:
* Files are detected in /data/encoded-video, /data/library, /data/upload
* Checked against the asset table
* Files are detected in /data/thumbs
@@ -69,20 +69,20 @@ export class IntegrityService extends BaseService {
@OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] })
async onConfigInit({
newConfig: {
integrityChecks: { orphanedFiles, missingFiles, checksumFiles },
integrityChecks: { untrackedFiles, missingFiles, checksumFiles },
},
}: ArgOf<'ConfigInit'>) {
this.integrityLock = await this.databaseRepository.tryLock(DatabaseLock.IntegrityCheck);
if (this.integrityLock) {
this.cronRepository.create({
name: 'integrityOrphanedFiles',
expression: orphanedFiles.cronExpression,
name: 'integrityUntrackedFiles',
expression: untrackedFiles.cronExpression,
onTick: () =>
handlePromiseError(
this.jobRepository.queue({ name: JobName.IntegrityOrphanedFilesQueueAll, data: {} }),
this.jobRepository.queue({ name: JobName.IntegrityUntrackedFilesQueueAll, data: {} }),
this.logger,
),
start: orphanedFiles.enabled,
start: untrackedFiles.enabled,
});
this.cronRepository.create({
@@ -109,7 +109,7 @@ export class IntegrityService extends BaseService {
@OnEvent({ name: 'ConfigUpdate', server: true })
onConfigUpdate({
newConfig: {
integrityChecks: { orphanedFiles, missingFiles, checksumFiles },
integrityChecks: { untrackedFiles, missingFiles, checksumFiles },
},
}: ArgOf<'ConfigUpdate'>) {
if (!this.integrityLock) {
@@ -117,9 +117,9 @@ export class IntegrityService extends BaseService {
}
this.cronRepository.update({
name: 'integrityOrphanedFiles',
expression: orphanedFiles.cronExpression,
start: orphanedFiles.enabled,
name: 'integrityUntrackedFiles',
expression: untrackedFiles.cronExpression,
start: untrackedFiles.enabled,
});
this.cronRepository.update({
@@ -194,16 +194,16 @@ export class IntegrityService extends BaseService {
}
}
@OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.IntegrityCheck })
async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
this.logger.log(`Checking for out of date orphaned file reports...`);
@OnJob({ name: JobName.IntegrityUntrackedFilesQueueAll, queue: QueueName.IntegrityCheck })
async handleUntrackedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
this.logger.log(`Checking for out of date untracked file reports...`);
const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReportType.OrphanFile);
const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReportType.UntrackedFile);
let total = 0;
for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) {
await this.jobRepository.queue({
name: JobName.IntegrityOrphanedFilesRefresh,
name: JobName.IntegrityUntrackedFilesRefresh,
data: {
items: batchReports,
},
@@ -218,7 +218,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
this.logger.log(`Scanning for orphaned files...`);
this.logger.log(`Scanning for untracked files...`);
const assetPaths = this.storageRepository.walk({
pathsToCrawl: [StorageFolder.EncodedVideo, StorageFolder.Library, StorageFolder.Upload].map((folder) =>
@@ -247,7 +247,7 @@ export class IntegrityService extends BaseService {
total = 0;
for await (const [batchType, batchPaths] of paths()) {
await this.jobRepository.queue({
name: JobName.IntegrityOrphanedFiles,
name: JobName.IntegrityUntrackedFiles,
data: {
type: batchType,
paths: batchPaths,
@@ -257,48 +257,48 @@ export class IntegrityService extends BaseService {
const count = batchPaths.length;
total += count;
this.logger.log(`Queued orphan check of ${count} file(s) (${total} so far)`);
this.logger.log(`Queued untracked check of ${count} file(s) (${total} so far)`);
}
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.IntegrityCheck })
async handleOrphanedFiles({ type, paths }: IIntegrityOrphanedFilesJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${paths.length} files to check if they are orphaned.`);
@OnJob({ name: JobName.IntegrityUntrackedFiles, queue: QueueName.IntegrityCheck })
async handleUntrackedFiles({ type, paths }: IIntegrityUntrackedFilesJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${paths.length} files to check if they are untracked.`);
const orphanedFiles = new Set<string>(paths);
const untrackedFiles = new Set<string>(paths);
if (type === 'asset') {
const assets = await this.integrityRepository.getAssetPathsByPaths(paths);
for (const { originalPath, encodedVideoPath } of assets) {
orphanedFiles.delete(originalPath);
untrackedFiles.delete(originalPath);
if (encodedVideoPath) {
orphanedFiles.delete(encodedVideoPath);
untrackedFiles.delete(encodedVideoPath);
}
}
} else {
const assets = await this.integrityRepository.getAssetFilePathsByPaths(paths);
for (const { path } of assets) {
orphanedFiles.delete(path);
untrackedFiles.delete(path);
}
}
if (orphanedFiles.size > 0) {
if (untrackedFiles.size > 0) {
await this.integrityRepository.create(
[...orphanedFiles].map((path) => ({
type: IntegrityReportType.OrphanFile,
[...untrackedFiles].map((path) => ({
type: IntegrityReportType.UntrackedFile,
path,
})),
);
}
this.logger.log(`Processed ${paths.length} and found ${orphanedFiles.size} orphaned file(s).`);
this.logger.log(`Processed ${paths.length} and found ${untrackedFiles.size} untracked file(s).`);
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityOrphanedFilesRefresh, queue: QueueName.IntegrityCheck })
async handleOrphanedRefresh({ items }: IIntegrityPathWithReportJob): Promise<JobStatus> {
@OnJob({ name: JobName.IntegrityUntrackedFilesRefresh, queue: QueueName.IntegrityCheck })
async handleUntrackedRefresh({ items }: IIntegrityPathWithReportJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${items.length} reports to check if they are out of date.`);
const results = await Promise.all(
@@ -619,7 +619,7 @@ export class IntegrityService extends BaseService {
properties = ['assetId', 'fileAssetId'] as const;
break;
}
case IntegrityReportType.OrphanFile: {
case IntegrityReportType.UntrackedFile: {
properties = [void 0] as const;
break;
}

12
server/src/services/job.service.ts
View File

@@ -38,8 +38,8 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
return { name: JobName.IntegrityMissingFilesQueueAll };
}
case ManualJobName.IntegrityOrphanFiles: {
return { name: JobName.IntegrityOrphanedFilesQueueAll };
case ManualJobName.IntegrityUntrackedFiles: {
return { name: JobName.IntegrityUntrackedFilesQueueAll };
}
case ManualJobName.IntegrityChecksumFiles: {
@@ -50,8 +50,8 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
return { name: JobName.IntegrityMissingFilesQueueAll, data: { refreshOnly: true } };
}
case ManualJobName.IntegrityOrphanFilesRefresh: {
return { name: JobName.IntegrityOrphanedFilesQueueAll, data: { refreshOnly: true } };
case ManualJobName.IntegrityUntrackedFilesRefresh: {
return { name: JobName.IntegrityUntrackedFilesQueueAll, data: { refreshOnly: true } };
}
case ManualJobName.IntegrityChecksumFilesRefresh: {
@@ -62,8 +62,8 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
return { name: JobName.IntegrityDeleteReportType, data: { type: IntegrityReportType.MissingFile } };
}
case ManualJobName.IntegrityOrphanFilesDeleteAll: {
return { name: JobName.IntegrityDeleteReportType, data: { type: IntegrityReportType.OrphanFile } };
case ManualJobName.IntegrityUntrackedFilesDeleteAll: {
return { name: JobName.IntegrityDeleteReportType, data: { type: IntegrityReportType.UntrackedFile } };
}
case ManualJobName.IntegrityChecksumFilesDeleteAll: {

2
server/src/services/system-config.service.spec.ts
View File

@@ -74,7 +74,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
tonemap: ToneMapping.Hable,
},
integrityChecks: {
orphanedFiles: {
untrackedFiles: {
enabled: true,
cronExpression: '0 03 * * *',
},