mirror of
https://github.com/immich-app/immich.git
synced 2026-03-22 23:19:04 +03:00
refactor: use medium tests for integrity service
Signed-off-by: izzy <me@insrt.uk>
This commit is contained in:
izzy
@@ -26,7 +26,7 @@ export class IntegrityRepository {
|
||||
}),
|
||||
)
|
||||
.returningAll()
|
||||
.executeTakeFirst();
|
||||
.executeTakeFirstOrThrow();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.STRING] })
|
||||
@@ -52,7 +52,7 @@ export class IntegrityRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ cursor: DummyValue.NUMBER, limit: 100 }, DummyValue.STRING] })
|
||||
async getIntegrityReports(pagination: ReportPaginationOptions, type: IntegrityReportType) {
|
||||
async getIntegrityReport(pagination: ReportPaginationOptions, type: IntegrityReportType) {
|
||||
const items = await this.db
|
||||
.selectFrom('integrity_report')
|
||||
.select(['id', 'type', 'path', 'assetId', 'fileAssetId', 'createdAt'])
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
import { text } from 'node:stream/consumers';
|
||||
import { AssetStatus, IntegrityReportType, JobName, JobStatus } from 'src/enum';
|
||||
import { IntegrityService } from 'src/services/integrity.service';
|
||||
import { makeStream, newTestService, ServiceMocks } from 'test/utils';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
|
||||
describe(IntegrityService.name, () => {
|
||||
let sut: IntegrityService;
|
||||
@@ -17,589 +13,11 @@ describe(IntegrityService.name, () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getIntegrityReportSummary', () => {
|
||||
it('gets summary', async () => {
|
||||
await sut.getIntegrityReportSummary();
|
||||
expect(mocks.integrityReport.getIntegrityReportSummary).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIntegrityReport', () => {
|
||||
it('gets report', async () => {
|
||||
mocks.integrityReport.getIntegrityReports.mockResolvedValue({
|
||||
items: [],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
|
||||
await expect(sut.getIntegrityReport({ type: IntegrityReportType.ChecksumFail })).resolves.toEqual({
|
||||
items: [],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.getIntegrityReports).toHaveBeenCalledWith(
|
||||
{ cursor: undefined, limit: 100 },
|
||||
IntegrityReportType.ChecksumFail,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIntegrityReportCsv', () => {
|
||||
it('gets report as csv', async () => {
|
||||
mocks.integrityReport.streamIntegrityReports.mockReturnValue(
|
||||
makeStream([
|
||||
{
|
||||
id: 'id',
|
||||
createdAt: new Date(0),
|
||||
path: '/path/to/file',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: null,
|
||||
fileAssetId: null,
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
await expect(text(sut.getIntegrityReportCsv(IntegrityReportType.ChecksumFail))).resolves.toMatchInlineSnapshot(`
|
||||
"id,type,assetId,fileAssetId,path
|
||||
id,checksum_mismatch,null,null,"/path/to/file"
|
||||
"
|
||||
`);
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReports).toHaveBeenCalledWith(IntegrityReportType.ChecksumFail);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIntegrityReportFile', () => {
|
||||
it('gets report file', async () => {
|
||||
mocks.integrityReport.getById.mockResolvedValue({
|
||||
id: 'id',
|
||||
createdAt: new Date(0),
|
||||
path: '/path/to/file',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: null,
|
||||
fileAssetId: null,
|
||||
});
|
||||
|
||||
await expect(sut.getIntegrityReportFile('id')).resolves.toEqual({
|
||||
path: '/path/to/file',
|
||||
fileName: 'file',
|
||||
contentType: 'application/octet-stream',
|
||||
cacheControl: 'private_without_cache',
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.getById).toHaveBeenCalledWith('id');
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteIntegrityReport', () => {
|
||||
it('deletes asset if one is present', async () => {
|
||||
mocks.integrityReport.getById.mockResolvedValue({
|
||||
id: 'id',
|
||||
createdAt: new Date(0),
|
||||
path: '/path/to/file',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: 'assetId',
|
||||
fileAssetId: null,
|
||||
});
|
||||
|
||||
await sut.deleteIntegrityReport('userId', 'id');
|
||||
|
||||
expect(mocks.asset.updateAll).toHaveBeenCalledWith(['assetId'], {
|
||||
deletedAt: expect.any(Date),
|
||||
status: AssetStatus.Trashed,
|
||||
});
|
||||
|
||||
expect(mocks.event.emit).toHaveBeenCalledWith('AssetTrashAll', {
|
||||
assetIds: ['assetId'],
|
||||
userId: 'userId',
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id');
|
||||
});
|
||||
|
||||
it('deletes file asset if one is present', async () => {
|
||||
mocks.integrityReport.getById.mockResolvedValue({
|
||||
id: 'id',
|
||||
createdAt: new Date(0),
|
||||
path: '/path/to/file',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: null,
|
||||
fileAssetId: 'fileAssetId',
|
||||
});
|
||||
|
||||
await sut.deleteIntegrityReport('userId', 'id');
|
||||
|
||||
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAssetId' }]);
|
||||
});
|
||||
|
||||
it('deletes untracked file', async () => {
|
||||
mocks.integrityReport.getById.mockResolvedValue({
|
||||
id: 'id',
|
||||
createdAt: new Date(0),
|
||||
path: '/path/to/file',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: null,
|
||||
fileAssetId: null,
|
||||
});
|
||||
|
||||
await sut.deleteIntegrityReport('userId', 'id');
|
||||
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/file');
|
||||
expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id');
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleUntrackedFilesQueueAll', () => {
|
||||
beforeEach(() => {
|
||||
mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never);
|
||||
});
|
||||
|
||||
it('queues jobs for all detected files', async () => {
|
||||
mocks.storage.walk.mockReturnValueOnce(makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']]));
|
||||
|
||||
mocks.storage.walk.mockReturnValueOnce(
|
||||
(function* () {
|
||||
yield ['/path/to/file3', '/path/to/file4'];
|
||||
yield ['/path/to/batch4'];
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleUntrackedFilesQueueAll({ refreshOnly: false });
|
||||
|
||||
expect(mocks.job.queue).toBeCalledTimes(4);
|
||||
expect(mocks.job.queue).toBeCalledWith({
|
||||
name: JobName.IntegrityUntrackedFiles,
|
||||
data: {
|
||||
type: 'asset',
|
||||
paths: expect.arrayContaining(['/path/to/file']),
|
||||
},
|
||||
});
|
||||
|
||||
expect(mocks.job.queue).toBeCalledWith({
|
||||
name: JobName.IntegrityUntrackedFiles,
|
||||
data: {
|
||||
type: 'asset_file',
|
||||
paths: expect.arrayContaining(['/path/to/file3']),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('queues jobs to refresh reports', async () => {
|
||||
mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue(
|
||||
(function* () {
|
||||
yield 'mockReport';
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleUntrackedFilesQueueAll({ refreshOnly: false });
|
||||
|
||||
expect(mocks.job.queue).toBeCalledTimes(1);
|
||||
expect(mocks.job.queue).toBeCalledWith({
|
||||
name: JobName.IntegrityUntrackedFilesRefresh,
|
||||
data: {
|
||||
items: expect.arrayContaining(['mockReport']),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleUntrackedFilesQueueAll({ refreshOnly: false })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleUntrackedFiles', () => {
|
||||
it('should detect untracked asset files', async () => {
|
||||
mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([
|
||||
{ originalPath: '/path/to/file1', encodedVideoPath: null },
|
||||
]);
|
||||
|
||||
await sut.handleUntrackedFiles({
|
||||
type: 'asset',
|
||||
paths: ['/path/to/file1', '/path/to/untracked'],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.getAssetPathsByPaths).toHaveBeenCalledWith(['/path/to/file1', '/path/to/untracked']);
|
||||
expect(mocks.integrityReport.create).toHaveBeenCalledWith([
|
||||
{ type: IntegrityReportType.UntrackedFile, path: '/path/to/untracked' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('should not create reports when no untracked files found for assets', async () => {
|
||||
mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([
|
||||
{ originalPath: '/path/to/file1', encodedVideoPath: '/path/to/encoded' },
|
||||
]);
|
||||
|
||||
await sut.handleUntrackedFiles({
|
||||
type: 'asset',
|
||||
paths: ['/path/to/file1', '/path/to/encoded'],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should detect untracked asset_file files', async () => {
|
||||
mocks.integrityReport.getAssetFilePathsByPaths.mockResolvedValue([{ path: '/path/to/thumb1' }]);
|
||||
|
||||
await sut.handleUntrackedFiles({
|
||||
type: 'asset_file',
|
||||
paths: ['/path/to/thumb1', '/path/to/untracked_thumb'],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.create).toHaveBeenCalledWith([
|
||||
{ type: IntegrityReportType.UntrackedFile, path: '/path/to/untracked_thumb' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleUntrackedRefresh', () => {
|
||||
it('should delete reports for files that no longer exist', async () => {
|
||||
mocks.storage.stat
|
||||
.mockRejectedValueOnce(new Error('ENOENT'))
|
||||
.mockResolvedValueOnce({} as never)
|
||||
.mockRejectedValueOnce(new Error('ENOENT'));
|
||||
|
||||
await sut.handleUntrackedRefresh({
|
||||
items: [
|
||||
{ reportId: 'report1', path: '/path/to/missing1' },
|
||||
{ reportId: 'report2', path: '/path/to/existing' },
|
||||
{ reportId: 'report3', path: '/path/to/missing2' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report1', 'report3']);
|
||||
});
|
||||
|
||||
it('should not delete reports for files that still exist', async () => {
|
||||
mocks.storage.stat.mockResolvedValue({} as never);
|
||||
|
||||
await sut.handleUntrackedRefresh({
|
||||
items: [{ reportId: 'report1', path: '/path/to/existing' }],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteByIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleUntrackedRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMissingFilesQueueAll', () => {
|
||||
beforeEach(() => {
|
||||
mocks.integrityReport.streamAssetPaths.mockReturnValue((function* () {})() as never);
|
||||
});
|
||||
|
||||
it('should queue jobs', async () => {
|
||||
mocks.integrityReport.streamAssetPaths.mockReturnValue(
|
||||
(function* () {
|
||||
yield { path: '/path/to/file1', assetId: 'asset1', fileAssetId: null };
|
||||
yield { path: '/path/to/file2', assetId: 'asset2', fileAssetId: null };
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleMissingFilesQueueAll({ refreshOnly: false });
|
||||
|
||||
expect(mocks.integrityReport.streamAssetPaths).toHaveBeenCalled();
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityMissingFiles,
|
||||
data: {
|
||||
items: expect.arrayContaining([{ path: '/path/to/file1', assetId: 'asset1', fileAssetId: null }]),
|
||||
},
|
||||
});
|
||||
|
||||
expect(mocks.job.queue).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: JobName.IntegrityMissingFilesRefresh,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should queue refresh jobs when refreshOnly is set', async () => {
|
||||
mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue(
|
||||
(function* () {
|
||||
yield { reportId: 'report1', path: '/path/to/file1' };
|
||||
yield { reportId: 'report2', path: '/path/to/file2' };
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleMissingFilesQueueAll({ refreshOnly: true });
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReportsWithAssetChecksum).toHaveBeenCalledWith(
|
||||
IntegrityReportType.MissingFile,
|
||||
);
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityMissingFilesRefresh,
|
||||
data: {
|
||||
items: expect.arrayContaining([{ reportId: 'report1', path: '/path/to/file1' }]),
|
||||
},
|
||||
});
|
||||
|
||||
expect(mocks.job.queue).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: JobName.IntegrityMissingFiles,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleMissingFilesQueueAll()).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMissingFiles', () => {
|
||||
it('should detect missing files and remove outdated reports', async () => {
|
||||
mocks.storage.stat
|
||||
.mockResolvedValueOnce({} as never)
|
||||
.mockRejectedValueOnce(new Error('ENOENT'))
|
||||
.mockResolvedValueOnce({} as never);
|
||||
|
||||
await sut.handleMissingFiles({
|
||||
items: [
|
||||
{ path: '/path/to/existing', assetId: 'asset1', fileAssetId: null, reportId: null },
|
||||
{ path: '/path/to/missing', assetId: 'asset2', fileAssetId: null, reportId: null },
|
||||
{ path: '/path/to/restored', assetId: 'asset3', fileAssetId: null, reportId: 'report2' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report2']);
|
||||
expect(mocks.integrityReport.create).toHaveBeenCalledWith([
|
||||
{ type: IntegrityReportType.MissingFile, path: '/path/to/missing', assetId: 'asset2', fileAssetId: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleMissingFiles({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMissingRefresh', () => {
|
||||
it('should remove outdated reports', async () => {
|
||||
mocks.storage.stat
|
||||
.mockResolvedValueOnce({} as never)
|
||||
.mockRejectedValueOnce(new Error('ENOENT'))
|
||||
.mockResolvedValueOnce({} as never);
|
||||
|
||||
await sut.handleMissingRefresh({
|
||||
items: [
|
||||
{ path: '/path/to/existing', reportId: null },
|
||||
{ path: '/path/to/missing', reportId: null },
|
||||
{ path: '/path/to/restored', reportId: 'report2' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report2']);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleMissingFiles({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleChecksumFiles', () => {
|
||||
beforeEach(() => {
|
||||
mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never);
|
||||
mocks.integrityReport.streamAssetChecksums.mockReturnValue((function* () {})() as never);
|
||||
mocks.integrityReport.getAssetCount.mockResolvedValue({ count: 1000 });
|
||||
mocks.systemMetadata.get.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
it('should queue refresh jobs when refreshOnly', async () => {
|
||||
mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue(
|
||||
(function* () {
|
||||
yield { reportId: 'report1', path: '/path/to/file1', checksum: Buffer.from('abc123', 'hex') };
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleChecksumFiles({ refreshOnly: true });
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReportsWithAssetChecksum).toHaveBeenCalledWith(
|
||||
IntegrityReportType.ChecksumFail,
|
||||
);
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityChecksumFilesRefresh,
|
||||
data: {
|
||||
items: [{ reportId: 'report1', path: '/path/to/file1', checksum: 'abc123' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should create report for checksum mismatch and delete when fixed', async () => {
|
||||
const fileContent = Buffer.from('test content');
|
||||
|
||||
mocks.integrityReport.streamAssetChecksums.mockReturnValue(
|
||||
(function* () {
|
||||
yield {
|
||||
originalPath: '/path/to/mismatch',
|
||||
checksum: 'mismatched checksum',
|
||||
createdAt: new Date(),
|
||||
assetId: 'asset1',
|
||||
reportId: null,
|
||||
};
|
||||
yield {
|
||||
originalPath: '/path/to/fixed',
|
||||
checksum: createHash('sha1').update(fileContent).digest(),
|
||||
createdAt: new Date(),
|
||||
assetId: 'asset2',
|
||||
reportId: 'report1',
|
||||
};
|
||||
})() as never,
|
||||
);
|
||||
|
||||
mocks.storage.createPlainReadStream.mockImplementation(() => Readable.from(fileContent));
|
||||
|
||||
await sut.handleChecksumFiles({ refreshOnly: false });
|
||||
|
||||
expect(mocks.integrityReport.create).toHaveBeenCalledWith({
|
||||
path: '/path/to/mismatch',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: 'asset1',
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('report1');
|
||||
});
|
||||
|
||||
it('should skip missing files', async () => {
|
||||
mocks.integrityReport.streamAssetChecksums.mockReturnValue(
|
||||
(function* () {
|
||||
yield {
|
||||
originalPath: '/path/to/missing',
|
||||
checksum: Buffer.from('abc', 'hex'),
|
||||
createdAt: new Date(),
|
||||
assetId: 'asset1',
|
||||
reportId: 'report1',
|
||||
};
|
||||
})() as never,
|
||||
);
|
||||
|
||||
const error = new Error('ENOENT') as NodeJS.ErrnoException;
|
||||
error.code = 'ENOENT';
|
||||
mocks.storage.createPlainReadStream.mockImplementation(() => {
|
||||
throw error;
|
||||
});
|
||||
|
||||
await sut.handleChecksumFiles({ refreshOnly: false });
|
||||
|
||||
expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('report1');
|
||||
expect(mocks.integrityReport.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleChecksumFiles({ refreshOnly: false })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleChecksumRefresh', () => {
|
||||
it('should delete reports when checksum now matches, file is missing, or asset is now missing', async () => {
|
||||
const fileContent = Buffer.from('test content');
|
||||
const correctChecksum = createHash('sha1').update(fileContent).digest().toString('hex');
|
||||
|
||||
const error = new Error('ENOENT') as NodeJS.ErrnoException;
|
||||
error.code = 'ENOENT';
|
||||
|
||||
mocks.storage.createPlainReadStream
|
||||
.mockImplementationOnce(() => Readable.from(fileContent))
|
||||
.mockImplementationOnce(() => {
|
||||
throw error;
|
||||
})
|
||||
.mockImplementationOnce(() => Readable.from(fileContent))
|
||||
.mockImplementationOnce(() => Readable.from(fileContent));
|
||||
|
||||
await sut.handleChecksumRefresh({
|
||||
items: [
|
||||
{ reportId: 'report1', path: '/path/to/fixed', checksum: correctChecksum },
|
||||
{ reportId: 'report2', path: '/path/to/missing', checksum: 'abc123' },
|
||||
{ reportId: 'report3', path: '/path/to/bad', checksum: 'wrongchecksum' },
|
||||
{ reportId: 'report4', path: '/path/to/missing-asset', checksum: null },
|
||||
],
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report1', 'report2', 'report4']);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleChecksumRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleDeleteAllIntegrityReports', () => {
|
||||
beforeEach(() => {
|
||||
mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue((function* () {})() as never);
|
||||
});
|
||||
|
||||
it('should queue delete jobs for checksum fail reports', async () => {
|
||||
mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue(
|
||||
(function* () {
|
||||
yield { id: 'report1', assetId: 'asset1', path: '/path/to/file1' };
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.ChecksumFail });
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(
|
||||
'assetId',
|
||||
IntegrityReportType.ChecksumFail,
|
||||
);
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityDeleteReports,
|
||||
data: {
|
||||
reports: [{ id: 'report1', assetId: 'asset1', path: '/path/to/file1' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should queue delete jobs for missing file reports by assetId and fileAssetId', async () => {
|
||||
mocks.integrityReport.streamIntegrityReportsByProperty
|
||||
.mockReturnValueOnce(
|
||||
(function* () {
|
||||
yield { id: 'report1', assetId: 'asset1', path: '/path/to/file1' };
|
||||
})() as never,
|
||||
)
|
||||
.mockReturnValueOnce(
|
||||
(function* () {
|
||||
yield { id: 'report2', fileAssetId: 'fileAsset1', path: '/path/to/file2' };
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.MissingFile });
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(
|
||||
'assetId',
|
||||
IntegrityReportType.MissingFile,
|
||||
);
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(
|
||||
'fileAssetId',
|
||||
IntegrityReportType.MissingFile,
|
||||
);
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should queue delete jobs for untracked file reports', async () => {
|
||||
mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue(
|
||||
(function* () {
|
||||
yield { id: 'report1', path: '/path/to/untracked' };
|
||||
})() as never,
|
||||
);
|
||||
|
||||
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.UntrackedFile });
|
||||
|
||||
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(
|
||||
undefined,
|
||||
IntegrityReportType.UntrackedFile,
|
||||
);
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityDeleteReports,
|
||||
data: {
|
||||
reports: [{ id: 'report1', path: '/path/to/untracked' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should query all property types when no type specified', async () => {
|
||||
await sut.handleDeleteAllIntegrityReports({});
|
||||
|
||||
@@ -607,45 +25,5 @@ describe(IntegrityService.name, () => {
|
||||
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith('assetId', undefined);
|
||||
expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith('fileAssetId', undefined);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleDeleteAllIntegrityReports({})).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleDeleteIntegrityReports', () => {
|
||||
it('should handle all report types', async () => {
|
||||
mocks.storage.unlink.mockResolvedValue(void 0);
|
||||
|
||||
await sut.handleDeleteIntegrityReports({
|
||||
reports: [
|
||||
{ id: 'report1', assetId: 'asset1', fileAssetId: null, path: '/path/to/file1' },
|
||||
{ id: 'report2', assetId: 'asset2', fileAssetId: null, path: '/path/to/file2' },
|
||||
{ id: 'report3', assetId: null, fileAssetId: 'fileAsset1', path: '/path/to/file3' },
|
||||
{ id: 'report4', assetId: null, fileAssetId: null, path: '/path/to/untracked' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(mocks.asset.updateAll).toHaveBeenCalledWith(['asset1', 'asset2'], {
|
||||
deletedAt: expect.any(Date),
|
||||
status: AssetStatus.Trashed,
|
||||
});
|
||||
|
||||
expect(mocks.event.emit).toHaveBeenCalledWith('AssetTrashAll', {
|
||||
assetIds: ['asset1', 'asset2'],
|
||||
userId: '',
|
||||
});
|
||||
|
||||
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report1', 'report2']);
|
||||
|
||||
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAsset1' }]);
|
||||
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/untracked');
|
||||
expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report4']);
|
||||
});
|
||||
|
||||
it('should succeed', async () => {
|
||||
await expect(sut.handleDeleteIntegrityReports({ reports: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -139,7 +139,7 @@ export class IntegrityService extends BaseService {
|
||||
}
|
||||
|
||||
getIntegrityReport(dto: IntegrityGetReportDto): Promise<IntegrityReportResponseDto> {
|
||||
return this.integrityRepository.getIntegrityReports({ cursor: dto.cursor, limit: dto.limit || 100 }, dto.type);
|
||||
return this.integrityRepository.getIntegrityReport({ cursor: dto.cursor, limit: dto.limit || 100 }, dto.type);
|
||||
}
|
||||
|
||||
getIntegrityReportCsv(type: IntegrityReportType): Readable {
|
||||
|
||||
@@ -28,6 +28,7 @@ import { CryptoRepository } from 'src/repositories/crypto.repository';
|
||||
import { DatabaseRepository } from 'src/repositories/database.repository';
|
||||
import { EmailRepository } from 'src/repositories/email.repository';
|
||||
import { EventRepository } from 'src/repositories/event.repository';
|
||||
import { IntegrityRepository } from 'src/repositories/integrity.repository';
|
||||
import { JobRepository } from 'src/repositories/job.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
|
||||
@@ -393,6 +394,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
|
||||
case AssetRepository:
|
||||
case AssetEditRepository:
|
||||
case AssetJobRepository:
|
||||
case IntegrityRepository:
|
||||
case MemoryRepository:
|
||||
case NotificationRepository:
|
||||
case OcrRepository:
|
||||
|
||||
990
server/test/medium/specs/services/integrity.service.spec.ts
Normal file
990
server/test/medium/specs/services/integrity.service.spec.ts
Normal file
@@ -0,0 +1,990 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
import { text } from 'node:stream/consumers';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { AssetFileType, IntegrityReportType, JobName, JobStatus } from 'src/enum';
|
||||
import { AssetRepository } from 'src/repositories/asset.repository';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { EventRepository } from 'src/repositories/event.repository';
|
||||
import { IntegrityRepository } from 'src/repositories/integrity.repository';
|
||||
import { JobRepository } from 'src/repositories/job.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { StorageRepository } from 'src/repositories/storage.repository';
|
||||
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
|
||||
import { DB } from 'src/schema';
|
||||
import { IntegrityService } from 'src/services/integrity.service';
|
||||
import { newMediumService } from 'test/medium.factory';
|
||||
import { getKyselyDB, makeStream } from 'test/utils';
|
||||
|
||||
let defaultDatabase: Kysely<DB>;
|
||||
|
||||
const setup = (db?: Kysely<DB>) => {
|
||||
return newMediumService(IntegrityService, {
|
||||
database: db || defaultDatabase,
|
||||
real: [IntegrityRepository, AssetRepository, ConfigRepository, SystemMetadataRepository],
|
||||
mock: [LoggingRepository, EventRepository, StorageRepository, JobRepository],
|
||||
});
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
defaultDatabase = await getKyselyDB();
|
||||
StorageCore.setMediaLocation('/path/to/file');
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
StorageCore.reset();
|
||||
});
|
||||
|
||||
describe(IntegrityService.name, () => {
|
||||
beforeEach(async () => {
|
||||
await defaultDatabase.deleteFrom('asset_file').execute();
|
||||
await defaultDatabase.deleteFrom('asset').execute();
|
||||
await defaultDatabase.deleteFrom('integrity_report').execute();
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
const { sut } = setup();
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getIntegrityReportSummary', () => {
|
||||
it('gets summary', async () => {
|
||||
const { sut } = setup();
|
||||
|
||||
await expect(sut.getIntegrityReportSummary()).resolves.toEqual({
|
||||
checksum_mismatch: 0,
|
||||
missing_file: 0,
|
||||
untracked_file: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIntegrityReport', () => {
|
||||
it('gets report', async () => {
|
||||
const { sut } = setup();
|
||||
|
||||
await expect(sut.getIntegrityReport({ type: IntegrityReportType.ChecksumFail })).resolves.toEqual({
|
||||
items: [],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIntegrityReportCsv', () => {
|
||||
it('gets report as csv', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
|
||||
const { id } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file',
|
||||
});
|
||||
|
||||
await expect(text(sut.getIntegrityReportCsv(IntegrityReportType.ChecksumFail))).resolves.toMatchInlineSnapshot(`
|
||||
"id,type,assetId,fileAssetId,path
|
||||
${id},checksum_mismatch,null,null,"/path/to/file"
|
||||
"
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIntegrityReportFile', () => {
|
||||
it('gets report file', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
|
||||
const { id } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file',
|
||||
});
|
||||
|
||||
await expect(sut.getIntegrityReportFile(id)).resolves.toEqual({
|
||||
path: '/path/to/file',
|
||||
fileName: 'file',
|
||||
contentType: 'application/octet-stream',
|
||||
cacheControl: 'private_without_cache',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteIntegrityReport', () => {
|
||||
it('deletes asset if one is present', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const events = ctx.getMock(EventRepository);
|
||||
events.emit.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId });
|
||||
|
||||
const { id } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file',
|
||||
assetId,
|
||||
});
|
||||
|
||||
await sut.deleteIntegrityReport(ownerId, id);
|
||||
|
||||
await expect(ctx.get(AssetRepository).getById(assetId)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
status: 'trashed',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(events.emit).toHaveBeenCalledWith('AssetTrashAll', {
|
||||
assetIds: [assetId],
|
||||
userId: ownerId,
|
||||
});
|
||||
|
||||
await expect(sut.getIntegrityReport({ type: IntegrityReportType.ChecksumFail })).resolves.toEqual({
|
||||
items: [],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('deletes file asset if one is present', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId });
|
||||
|
||||
const fileAssetId = randomUUID();
|
||||
await ctx.newAssetFile({ id: fileAssetId, assetId, type: AssetFileType.Thumbnail, path: '/path/to/file' });
|
||||
|
||||
const { id } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file',
|
||||
fileAssetId,
|
||||
});
|
||||
|
||||
await sut.deleteIntegrityReport('userId', id);
|
||||
|
||||
await expect(ctx.get(AssetRepository).getForThumbnail(assetId, AssetFileType.Thumbnail, false)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
path: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('deletes untracked file', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
storage.unlink.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: userId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const { id } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file',
|
||||
});
|
||||
|
||||
await sut.deleteIntegrityReport(userId, id);
|
||||
|
||||
expect(storage.unlink).toHaveBeenCalledWith('/path/to/file');
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleUntrackedFilesQueueAll', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
storage.walk.mockImplementation(() => makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']]));
|
||||
|
||||
await expect(sut.handleUntrackedFilesQueueAll({ refreshOnly: false })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('queues jobs for all detected files', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
storage.walk.mockReturnValueOnce(makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']]));
|
||||
storage.walk.mockReturnValueOnce(makeStream([['/path/to/file3', '/path/to/file4'], ['/path/to/batch4']]));
|
||||
|
||||
await sut.handleUntrackedFilesQueueAll({ refreshOnly: false });
|
||||
|
||||
expect(job.queue).toBeCalledTimes(4);
|
||||
expect(job.queue).toBeCalledWith({
|
||||
name: JobName.IntegrityUntrackedFiles,
|
||||
data: {
|
||||
type: 'asset',
|
||||
paths: expect.arrayContaining(['/path/to/file']),
|
||||
},
|
||||
});
|
||||
|
||||
expect(job.queue).toBeCalledWith({
|
||||
name: JobName.IntegrityUntrackedFiles,
|
||||
data: {
|
||||
type: 'asset_file',
|
||||
paths: expect.arrayContaining(['/path/to/file3']),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('queues jobs to refresh reports', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
storage.walk.mockImplementation(() => makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']]));
|
||||
|
||||
const { id } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.UntrackedFile,
|
||||
path: '/path/to/file',
|
||||
});
|
||||
|
||||
await sut.handleUntrackedFilesQueueAll({ refreshOnly: true });
|
||||
|
||||
expect(job.queue).toBeCalledTimes(1);
|
||||
expect(job.queue).toBeCalledWith({
|
||||
name: JobName.IntegrityUntrackedFilesRefresh,
|
||||
data: {
|
||||
items: expect.arrayContaining([
|
||||
{
|
||||
path: '/path/to/file',
|
||||
reportId: id,
|
||||
},
|
||||
]),
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleUntrackedFiles', () => {
|
||||
it('should detect untracked asset files', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', encodedVideoPath: '/path/to/file2' });
|
||||
|
||||
await sut.handleUntrackedFiles({
|
||||
type: 'asset',
|
||||
paths: ['/path/to/file1', '/path/to/file2', '/path/to/untracked'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.UntrackedFile,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
path: '/path/to/untracked',
|
||||
}),
|
||||
],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('should detect untracked asset_file files', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId });
|
||||
|
||||
await ctx.newAssetFile({ assetId, type: AssetFileType.Thumbnail, path: '/path/to/file1' });
|
||||
|
||||
await sut.handleUntrackedFiles({
|
||||
type: 'asset_file',
|
||||
paths: ['/path/to/file1', '/path/to/untracked'],
|
||||
});
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.UntrackedFile,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
path: '/path/to/untracked',
|
||||
}),
|
||||
],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleUntrackedRefresh', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleUntrackedRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should delete reports for files that no longer exist', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
|
||||
const report1 = await integrity.create({
|
||||
type: IntegrityReportType.UntrackedFile,
|
||||
path: '/path/to/missing1',
|
||||
});
|
||||
|
||||
const report2 = await integrity.create({
|
||||
type: IntegrityReportType.UntrackedFile,
|
||||
path: '/path/to/existing',
|
||||
});
|
||||
|
||||
storage.stat.mockRejectedValueOnce(new Error('ENOENT')).mockResolvedValueOnce({} as never);
|
||||
|
||||
await sut.handleUntrackedRefresh({
|
||||
items: [
|
||||
{ reportId: report1.id, path: report1.path },
|
||||
{ reportId: report2.id, path: report2.path },
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.UntrackedFile,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
path: '/path/to/existing',
|
||||
}),
|
||||
],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMissingFilesQueueAll', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleMissingFilesQueueAll()).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should queue jobs', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' });
|
||||
|
||||
const {
|
||||
result: { id: assetId2 },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file2' });
|
||||
|
||||
const { id: reportId } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.UntrackedFile,
|
||||
path: '/path/to/file2',
|
||||
assetId: assetId2,
|
||||
});
|
||||
|
||||
await sut.handleMissingFilesQueueAll({ refreshOnly: false });
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityMissingFiles,
|
||||
data: {
|
||||
items: expect.arrayContaining([
|
||||
{ path: '/path/to/file1', assetId, fileAssetId: null, reportId: null },
|
||||
{ path: '/path/to/file2', assetId: assetId2, fileAssetId: null, reportId },
|
||||
]),
|
||||
},
|
||||
});
|
||||
|
||||
expect(job.queue).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: JobName.IntegrityMissingFilesRefresh,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should queue refresh jobs when refreshOnly is set', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const { id: reportId } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.MissingFile,
|
||||
path: '/path/to/file1',
|
||||
});
|
||||
|
||||
await sut.handleMissingFilesQueueAll({ refreshOnly: true });
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityMissingFilesRefresh,
|
||||
data: {
|
||||
items: expect.arrayContaining([{ reportId, path: '/path/to/file1' }]),
|
||||
},
|
||||
});
|
||||
|
||||
expect(job.queue).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: JobName.IntegrityMissingFiles,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMissingFiles', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleMissingFiles({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should detect missing files and remove outdated reports', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
|
||||
const { id: restoredId } = await integrity.create({
|
||||
type: IntegrityReportType.MissingFile,
|
||||
path: '/path/to/restored',
|
||||
});
|
||||
|
||||
storage.stat
|
||||
.mockResolvedValueOnce({} as never)
|
||||
.mockRejectedValueOnce(new Error('ENOENT'))
|
||||
.mockResolvedValueOnce({} as never);
|
||||
|
||||
await sut.handleMissingFiles({
|
||||
items: [
|
||||
{ path: '/path/to/existing', assetId: null, fileAssetId: null, reportId: null },
|
||||
{ path: '/path/to/missing', assetId: null, fileAssetId: null, reportId: null },
|
||||
{ path: '/path/to/restored', assetId: null, fileAssetId: null, reportId: restoredId },
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.MissingFile,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
path: '/path/to/missing',
|
||||
}),
|
||||
],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMissingRefresh', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleMissingRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should remove outdated reports', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
|
||||
const { id: restoredId } = await integrity.create({
|
||||
type: IntegrityReportType.MissingFile,
|
||||
path: '/path/to/restored',
|
||||
});
|
||||
|
||||
storage.stat
|
||||
.mockResolvedValueOnce({} as never)
|
||||
.mockRejectedValueOnce(new Error('ENOENT'))
|
||||
.mockResolvedValueOnce({} as never);
|
||||
|
||||
await sut.handleMissingRefresh({
|
||||
items: [
|
||||
{ path: '/path/to/existing', reportId: null },
|
||||
{ path: '/path/to/missing', reportId: null },
|
||||
{ path: '/path/to/restored', reportId: restoredId },
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.MissingFile,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleChecksumFiles', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleChecksumFiles({ refreshOnly: false })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should queue refresh jobs when refreshOnly', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', checksum: Buffer.from('a') });
|
||||
|
||||
const { id: reportId } = await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file1',
|
||||
assetId,
|
||||
});
|
||||
|
||||
await sut.handleChecksumFiles({ refreshOnly: true });
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityChecksumFilesRefresh,
|
||||
data: {
|
||||
items: [{ reportId, path: '/path/to/file1', checksum: '61' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should create report for checksum mismatch and delete when fixed', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', checksum: Buffer.from('mismatched') });
|
||||
|
||||
const fileContent1 = Buffer.from('test content');
|
||||
await ctx.newAsset({
|
||||
ownerId,
|
||||
originalPath: '/path/to/file2',
|
||||
checksum: createHash('sha1').update(fileContent1).digest(),
|
||||
});
|
||||
|
||||
const fileContent2 = Buffer.from('test content 2');
|
||||
const {
|
||||
result: { id: assetId3 },
|
||||
} = await ctx.newAsset({
|
||||
ownerId,
|
||||
originalPath: '/path/to/file3',
|
||||
checksum: createHash('sha1').update(fileContent2).digest(),
|
||||
});
|
||||
|
||||
await ctx.get(IntegrityRepository).create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file3',
|
||||
assetId: assetId3,
|
||||
});
|
||||
|
||||
storage.createPlainReadStream.mockImplementation((path) =>
|
||||
Readable.from(
|
||||
path === '/path/to/file2' ? fileContent1 : path === '/path/to/file3' ? fileContent2 : 'garbage data',
|
||||
),
|
||||
);
|
||||
|
||||
await sut.handleChecksumFiles({ refreshOnly: false });
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.ChecksumFail,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
assetId,
|
||||
path: '/path/to/file1',
|
||||
}),
|
||||
],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('should skip missing files', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', checksum: Buffer.from('a') });
|
||||
|
||||
const error = new Error('ENOENT') as NodeJS.ErrnoException;
|
||||
error.code = 'ENOENT';
|
||||
storage.createPlainReadStream.mockImplementation(() => {
|
||||
throw error;
|
||||
});
|
||||
|
||||
await sut.handleChecksumFiles({ refreshOnly: false });
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.ChecksumFail,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleChecksumRefresh', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleChecksumRefresh({ items: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should delete reports when checksum now matches, file is missing, or asset is now missing', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const fileContent = Buffer.from('test content');
|
||||
const correctChecksum = createHash('sha1').update(fileContent).digest();
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: fixedAssetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/fixed', checksum: correctChecksum });
|
||||
|
||||
const { id: fixedReportId } = await integrity.create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/fixed',
|
||||
assetId: fixedAssetId,
|
||||
});
|
||||
|
||||
const {
|
||||
result: { id: missingAssetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/missing', checksum: Buffer.from('1') });
|
||||
|
||||
const { id: missingReportId } = await integrity.create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/missing',
|
||||
assetId: missingAssetId,
|
||||
});
|
||||
|
||||
const {
|
||||
result: { id: badAssetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/missing', checksum: Buffer.from('2') });
|
||||
|
||||
const { id: badReportId } = await integrity.create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/bad',
|
||||
assetId: badAssetId,
|
||||
});
|
||||
|
||||
const { id: missingAssetReportId } = await integrity.create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/missing-asset',
|
||||
});
|
||||
|
||||
const error = new Error('ENOENT') as NodeJS.ErrnoException;
|
||||
error.code = 'ENOENT';
|
||||
|
||||
storage.createPlainReadStream
|
||||
.mockImplementationOnce(() => Readable.from(fileContent))
|
||||
.mockImplementationOnce(() => {
|
||||
throw error;
|
||||
})
|
||||
.mockImplementationOnce(() => Readable.from(fileContent))
|
||||
.mockImplementationOnce(() => Readable.from(fileContent));
|
||||
|
||||
await sut.handleChecksumRefresh({
|
||||
items: [
|
||||
{ reportId: fixedReportId, path: '/path/to/fixed', checksum: correctChecksum.toString('hex') },
|
||||
{ reportId: missingReportId, path: '/path/to/missing', checksum: 'abc123' },
|
||||
{ reportId: badReportId, path: '/path/to/bad', checksum: 'wrongchecksum' },
|
||||
{ reportId: missingAssetReportId, path: '/path/to/missing-asset', checksum: null },
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
ctx.get(IntegrityRepository).getIntegrityReport(
|
||||
{
|
||||
limit: 100,
|
||||
},
|
||||
IntegrityReportType.ChecksumFail,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
id: badReportId,
|
||||
assetId: badAssetId,
|
||||
path: '/path/to/bad',
|
||||
}),
|
||||
],
|
||||
nextCursor: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleDeleteAllIntegrityReports', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleDeleteAllIntegrityReports({})).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should queue delete jobs for checksum fail reports', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' });
|
||||
|
||||
const { id: reportId } = await integrity.create({
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
path: '/path/to/file1',
|
||||
assetId,
|
||||
});
|
||||
|
||||
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.ChecksumFail });
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityDeleteReports,
|
||||
data: {
|
||||
reports: [{ id: reportId, assetId, path: '/path/to/file1', fileAssetId: null }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should queue delete jobs for missing file reports by assetId and fileAssetId', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' });
|
||||
|
||||
const { id: assetReportId } = await integrity.create({
|
||||
type: IntegrityReportType.MissingFile,
|
||||
path: '/path/to/file1',
|
||||
assetId,
|
||||
});
|
||||
|
||||
const {
|
||||
result: { id: assetId2 },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file2' });
|
||||
|
||||
const fileAssetId = randomUUID();
|
||||
await ctx.newAssetFile({
|
||||
id: fileAssetId,
|
||||
assetId: assetId2,
|
||||
path: '/path/to/file3',
|
||||
type: AssetFileType.Thumbnail,
|
||||
});
|
||||
|
||||
const { id: fileAssetReportId } = await integrity.create({
|
||||
type: IntegrityReportType.MissingFile,
|
||||
path: '/path/to/file3',
|
||||
fileAssetId,
|
||||
});
|
||||
|
||||
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.MissingFile });
|
||||
|
||||
expect(job.queue).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityDeleteReports,
|
||||
data: {
|
||||
reports: [{ id: assetReportId, assetId, path: '/path/to/file1', fileAssetId: null }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityDeleteReports,
|
||||
data: {
|
||||
reports: [{ id: fileAssetReportId, assetId: null, path: '/path/to/file3', fileAssetId }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should queue delete jobs for untracked file reports', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const job = ctx.getMock(JobRepository);
|
||||
job.queue.mockResolvedValue(void 0);
|
||||
|
||||
const { id: reportId } = await integrity.create({
|
||||
type: IntegrityReportType.UntrackedFile,
|
||||
path: '/path/to/untracked',
|
||||
});
|
||||
|
||||
await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.UntrackedFile });
|
||||
|
||||
expect(job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.IntegrityDeleteReports,
|
||||
data: {
|
||||
reports: [{ id: reportId, path: '/path/to/untracked', assetId: null, fileAssetId: null }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleDeleteIntegrityReports', () => {
|
||||
it('should succeed', async () => {
|
||||
const { sut } = setup();
|
||||
await expect(sut.handleDeleteIntegrityReports({ reports: [] })).resolves.toBe(JobStatus.Success);
|
||||
});
|
||||
|
||||
it('should handle all report types', async () => {
|
||||
const { sut, ctx } = setup();
|
||||
const integrity = ctx.get(IntegrityRepository);
|
||||
const storage = ctx.getMock(StorageRepository);
|
||||
const events = ctx.getMock(EventRepository);
|
||||
|
||||
storage.unlink.mockResolvedValue(void 0);
|
||||
events.emit.mockResolvedValue(void 0);
|
||||
|
||||
const {
|
||||
result: { id: ownerId },
|
||||
} = await ctx.newUser();
|
||||
|
||||
const {
|
||||
result: { id: assetId1 },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' });
|
||||
|
||||
const { id: reportId1 } = await integrity.create({
|
||||
path: '/path/to/file1',
|
||||
type: IntegrityReportType.ChecksumFail,
|
||||
assetId: assetId1,
|
||||
});
|
||||
|
||||
const {
|
||||
result: { id: assetId2 },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file2' });
|
||||
|
||||
const { id: reportId2 } = await integrity.create({
|
||||
path: '/path/to/file2',
|
||||
type: IntegrityReportType.MissingFile,
|
||||
assetId: assetId2,
|
||||
});
|
||||
|
||||
const {
|
||||
result: { id: assetId3 },
|
||||
} = await ctx.newAsset({ ownerId, originalPath: '/path/to/file3' });
|
||||
|
||||
const fileAssetId = randomUUID();
|
||||
await ctx.newAssetFile({
|
||||
id: fileAssetId,
|
||||
assetId: assetId3,
|
||||
path: '/path/to/file4',
|
||||
type: AssetFileType.Thumbnail,
|
||||
});
|
||||
|
||||
const { id: reportId3 } = await integrity.create({
|
||||
path: '/path/to/file4',
|
||||
type: IntegrityReportType.MissingFile,
|
||||
fileAssetId,
|
||||
});
|
||||
|
||||
const { id: reportId4 } = await integrity.create({
|
||||
path: '/path/to/untracked',
|
||||
type: IntegrityReportType.UntrackedFile,
|
||||
});
|
||||
|
||||
await sut.handleDeleteIntegrityReports({
|
||||
reports: [
|
||||
{ id: reportId1, assetId: assetId1, fileAssetId: null, path: '/path/to/file1' },
|
||||
{ id: reportId2, assetId: assetId2, fileAssetId: null, path: '/path/to/file2' },
|
||||
{ id: reportId3, assetId: null, fileAssetId, path: '/path/to/file4' },
|
||||
{ id: reportId4, assetId: null, fileAssetId: null, path: '/path/to/untracked' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(events.emit).toHaveBeenCalledWith('AssetTrashAll', {
|
||||
assetIds: [assetId1, assetId2],
|
||||
userId: '',
|
||||
});
|
||||
|
||||
expect(storage.unlink).toHaveBeenCalledWith('/path/to/untracked');
|
||||
|
||||
await expect(ctx.get(AssetRepository).getByIds([assetId1, assetId2, assetId3])).resolves.toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: assetId1,
|
||||
status: 'trashed',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: assetId2,
|
||||
status: 'trashed',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: assetId3,
|
||||
status: 'active',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
await expect(defaultDatabase.selectFrom('asset_file').execute()).resolves.toEqual([]);
|
||||
await expect(defaultDatabase.selectFrom('integrity_report').execute()).resolves.toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user