merge: remote-tracking branch 'origin/main' into feat/integrity-checks-izzy

This commit is contained in:
izzy
2026-01-21 17:02:22 +00:00
310 changed files with 18273 additions and 3199 deletions

View File

@@ -107,6 +107,78 @@ describe(ApiKeyService.name, () => {
permissions: newPermissions,
});
});
describe('api key auth', () => {
it('should prevent adding Permission.all', async () => {
const permissions = [Permission.ApiKeyCreate, Permission.ApiKeyUpdate, Permission.AssetRead];
const auth = factory.auth({ apiKey: { permissions } });
const apiKey = factory.apiKey({ userId: auth.user.id, permissions });
mocks.apiKey.getById.mockResolvedValue(apiKey);
await expect(sut.update(auth, apiKey.id, { permissions: [Permission.All] })).rejects.toThrow(
'Cannot grant permissions you do not have',
);
expect(mocks.apiKey.update).not.toHaveBeenCalled();
});
it('should prevent adding a new permission', async () => {
const permissions = [Permission.ApiKeyCreate, Permission.ApiKeyUpdate, Permission.AssetRead];
const auth = factory.auth({ apiKey: { permissions } });
const apiKey = factory.apiKey({ userId: auth.user.id, permissions });
mocks.apiKey.getById.mockResolvedValue(apiKey);
await expect(sut.update(auth, apiKey.id, { permissions: [Permission.AssetCopy] })).rejects.toThrow(
'Cannot grant permissions you do not have',
);
expect(mocks.apiKey.update).not.toHaveBeenCalled();
});
it('should allow removing permissions', async () => {
const auth = factory.auth({ apiKey: { permissions: [Permission.ApiKeyUpdate, Permission.AssetRead] } });
const apiKey = factory.apiKey({
userId: auth.user.id,
permissions: [Permission.AssetRead, Permission.AssetDelete],
});
mocks.apiKey.getById.mockResolvedValue(apiKey);
mocks.apiKey.update.mockResolvedValue(apiKey);
// remove Permission.AssetDelete
await sut.update(auth, apiKey.id, { permissions: [Permission.AssetRead] });
expect(mocks.apiKey.update).toHaveBeenCalledWith(
auth.user.id,
apiKey.id,
expect.objectContaining({ permissions: [Permission.AssetRead] }),
);
});
it('should allow adding new permissions', async () => {
const auth = factory.auth({
apiKey: { permissions: [Permission.ApiKeyUpdate, Permission.AssetRead, Permission.AssetUpdate] },
});
const apiKey = factory.apiKey({ userId: auth.user.id, permissions: [Permission.AssetRead] });
mocks.apiKey.getById.mockResolvedValue(apiKey);
mocks.apiKey.update.mockResolvedValue(apiKey);
// add Permission.AssetUpdate
await sut.update(auth, apiKey.id, {
name: apiKey.name,
permissions: [Permission.AssetRead, Permission.AssetUpdate],
});
expect(mocks.apiKey.update).toHaveBeenCalledWith(
auth.user.id,
apiKey.id,
expect.objectContaining({ permissions: [Permission.AssetRead, Permission.AssetUpdate] }),
);
});
});
});
describe('delete', () => {

View File

@@ -32,6 +32,14 @@ export class ApiKeyService extends BaseService {
throw new BadRequestException('API Key not found');
}
if (
auth.apiKey &&
dto.permissions &&
!isGranted({ requested: dto.permissions, current: auth.apiKey.permissions })
) {
throw new BadRequestException('Cannot grant permissions you do not have');
}
const key = await this.apiKeyRepository.update(auth.user.id, id, { name: dto.name, permissions: dto.permissions });
return this.map(key);

View File

@@ -529,9 +529,10 @@ describe(AssetMediaService.name, () => {
...assetStub.withCropEdit.files,
{
id: 'edited-file',
type: AssetFileType.FullSizeEdited,
type: AssetFileType.FullSize,
path: '/uploads/user-id/fullsize/edited.jpg',
} as AssetFile,
isEdited: true,
},
],
};
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
@@ -554,9 +555,10 @@ describe(AssetMediaService.name, () => {
...assetStub.withCropEdit.files,
{
id: 'edited-file',
type: AssetFileType.FullSizeEdited,
type: AssetFileType.FullSize,
path: '/uploads/user-id/fullsize/edited.jpg',
} as AssetFile,
isEdited: true,
},
],
};
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
@@ -579,9 +581,10 @@ describe(AssetMediaService.name, () => {
...assetStub.withCropEdit.files,
{
id: 'edited-file',
type: AssetFileType.FullSizeEdited,
type: AssetFileType.FullSize,
path: '/uploads/user-id/fullsize/edited.jpg',
} as AssetFile,
isEdited: true,
},
],
};
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
@@ -656,6 +659,7 @@ describe(AssetMediaService.name, () => {
id: '42',
path: '/path/to/preview',
type: AssetFileType.Thumbnail,
isEdited: false,
},
],
});
@@ -673,6 +677,7 @@ describe(AssetMediaService.name, () => {
id: '42',
path: '/path/to/preview.jpg',
type: AssetFileType.Preview,
isEdited: false,
},
],
});

View File

@@ -4,7 +4,7 @@ import { DateTime, Duration } from 'luxon';
import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { AssetFile } from 'src/database';
import { OnJob } from 'src/decorators';
import { AssetResponseDto, MapAsset, SanitizedAssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
import { AssetResponseDto, SanitizedAssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
import {
AssetBulkDeleteDto,
AssetBulkUpdateDto,
@@ -112,7 +112,7 @@ export class AssetService extends BaseService {
const { description, dateTimeOriginal, latitude, longitude, rating, ...rest } = dto;
const repos = { asset: this.assetRepository, event: this.eventRepository };
let previousMotion: MapAsset | null = null;
let previousMotion: { id: string } | null = null;
if (rest.livePhotoVideoId) {
await onBeforeLink(repos, { userId: auth.user.id, livePhotoVideoId: rest.livePhotoVideoId });
} else if (rest.livePhotoVideoId === null) {

View File

@@ -5,7 +5,7 @@ import { StorageCore } from 'src/cores/storage.core';
import { ImmichWorker, JobStatus, StorageFolder } from 'src/enum';
import { BackupService } from 'src/services/backup.service';
import { systemConfigStub } from 'test/fixtures/system-config.stub';
import { mockSpawn, newTestService, ServiceMocks } from 'test/utils';
import { mockDuplex, mockSpawn, newTestService, ServiceMocks } from 'test/utils';
import { describe } from 'vitest';
describe(BackupService.name, () => {
@@ -147,6 +147,7 @@ describe(BackupService.name, () => {
beforeEach(() => {
mocks.storage.readdir.mockResolvedValue([]);
mocks.process.spawn.mockReturnValue(mockSpawn(0, 'data', ''));
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex('command', 0, 'data', ''));
mocks.storage.rename.mockResolvedValue();
mocks.storage.unlink.mockResolvedValue();
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
@@ -165,7 +166,7 @@ describe(BackupService.name, () => {
({ sut, mocks } = newTestService(BackupService, { config: configMock }));
mocks.storage.readdir.mockResolvedValue([]);
mocks.process.spawn.mockReturnValue(mockSpawn(0, 'data', ''));
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex('command', 0, 'data', ''));
mocks.storage.rename.mockResolvedValue();
mocks.storage.unlink.mockResolvedValue();
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
@@ -174,14 +175,16 @@ describe(BackupService.name, () => {
await sut.handleBackupDatabase();
expect(mocks.process.spawn).toHaveBeenCalled();
const call = mocks.process.spawn.mock.calls[0];
expect(mocks.process.spawnDuplexStream).toHaveBeenCalled();
const call = mocks.process.spawnDuplexStream.mock.calls[0];
const args = call[1] as string[];
// ['--dbname', '<url>', '--clean', '--if-exists']
expect(args[0]).toBe('--dbname');
const passedUrl = args[1];
expect(passedUrl).not.toContain('uselibpqcompat');
expect(passedUrl).toContain('sslmode=require');
expect(args).toMatchInlineSnapshot(`
[
"postgresql://postgres:pwd@host:5432/immich?sslmode=require",
"--clean",
"--if-exists",
]
`);
});
it('should run a database backup successfully', async () => {
@@ -196,21 +199,21 @@ describe(BackupService.name, () => {
expect(mocks.storage.rename).toHaveBeenCalled();
});
it('should fail if pg_dumpall fails', async () => {
mocks.process.spawn.mockReturnValueOnce(mockSpawn(1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('Backup failed with code 1');
it('should fail if pg_dump fails', async () => {
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
});
it('should not rename file if pgdump fails and gzip succeeds', async () => {
mocks.process.spawn.mockReturnValueOnce(mockSpawn(1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('Backup failed with code 1');
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
expect(mocks.storage.rename).not.toHaveBeenCalled();
});
it('should fail if gzip fails', async () => {
mocks.process.spawn.mockReturnValueOnce(mockSpawn(0, 'data', ''));
mocks.process.spawn.mockReturnValueOnce(mockSpawn(1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('Gzip failed with code 1');
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 0, 'data', ''));
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('gzip', 1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('gzip non-zero exit code (1)');
});
it('should fail if write stream fails', async () => {
@@ -226,9 +229,9 @@ describe(BackupService.name, () => {
});
it('should ignore unlink failing and still return failed job status', async () => {
mocks.process.spawn.mockReturnValueOnce(mockSpawn(1, '', 'error'));
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 1, '', 'error'));
mocks.storage.unlink.mockRejectedValue(new Error('error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('Backup failed with code 1');
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
expect(mocks.storage.unlink).toHaveBeenCalled();
});
@@ -242,12 +245,12 @@ describe(BackupService.name, () => {
${'17.15.1'} | ${17}
${'18.0.0'} | ${18}
`(
`should use pg_dumpall $expectedVersion with postgres version $postgresVersion`,
`should use pg_dump $expectedVersion with postgres version $postgresVersion`,
async ({ postgresVersion, expectedVersion }) => {
mocks.database.getPostgresVersion.mockResolvedValue(postgresVersion);
await sut.handleBackupDatabase();
expect(mocks.process.spawn).toHaveBeenCalledWith(
`/usr/lib/postgresql/${expectedVersion}/bin/pg_dumpall`,
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledWith(
`/usr/lib/postgresql/${expectedVersion}/bin/pg_dump`,
expect.any(Array),
expect.any(Object),
);

View File

@@ -1,13 +1,16 @@
import { Injectable } from '@nestjs/common';
import { DateTime } from 'luxon';
import path from 'node:path';
import semver from 'semver';
import { serverVersion } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent, OnJob } from 'src/decorators';
import { DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum';
import { ArgOf } from 'src/repositories/event.repository';
import { BaseService } from 'src/services/base.service';
import {
createDatabaseBackup,
isFailedDatabaseBackupName,
isValidDatabaseRoutineBackupName,
UnsupportedPostgresError,
} from 'src/utils/database-backups';
import { handlePromiseError } from 'src/utils/misc';
@Injectable()
@@ -53,16 +56,11 @@ export class BackupService extends BaseService {
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
const files = await this.storageRepository.readdir(backupsFolder);
const failedBackups = files.filter((file) => file.match(/immich-db-backup-.*\.sql\.gz\.tmp$/));
const backups = files
.filter((file) => {
const oldBackupStyle = file.match(/immich-db-backup-\d+\.sql\.gz$/);
//immich-db-backup-20250729T114018-v1.136.0-pg14.17.sql.gz
const newBackupStyle = file.match(/immich-db-backup-\d{8}T\d{6}-v.*-pg.*\.sql\.gz$/);
return oldBackupStyle || newBackupStyle;
})
.filter((filename) => isValidDatabaseRoutineBackupName(filename))
.toSorted()
.toReversed();
const failedBackups = files.filter((filename) => isFailedDatabaseBackupName(filename));
const toDelete = backups.slice(config.keepLastAmount);
toDelete.push(...failedBackups);
@@ -75,123 +73,27 @@ export class BackupService extends BaseService {
@OnJob({ name: JobName.DatabaseBackup, queue: QueueName.BackupDatabase })
async handleBackupDatabase(): Promise<JobStatus> {
this.logger.debug(`Database Backup Started`);
const { database } = this.configRepository.getEnv();
const config = database.config;
const isUrlConnection = config.connectionType === 'url';
let connectionUrl: string = isUrlConnection ? config.url : '';
if (URL.canParse(connectionUrl)) {
// remove known bad url parameters for pg_dumpall
const url = new URL(connectionUrl);
url.searchParams.delete('uselibpqcompat');
connectionUrl = url.toString();
}
const databaseParams = isUrlConnection
? ['--dbname', connectionUrl]
: [
'--username',
config.username,
'--host',
config.host,
'--port',
`${config.port}`,
'--database',
config.database,
];
databaseParams.push('--clean', '--if-exists');
const databaseVersion = await this.databaseRepository.getPostgresVersion();
const backupFilePath = path.join(
StorageCore.getBaseFolder(StorageFolder.Backups),
`immich-db-backup-${DateTime.now().toFormat("yyyyLLdd'T'HHmmss")}-v${serverVersion.toString()}-pg${databaseVersion.split(' ')[0]}.sql.gz.tmp`,
);
const databaseSemver = semver.coerce(databaseVersion);
const databaseMajorVersion = databaseSemver?.major;
if (!databaseMajorVersion || !databaseSemver || !semver.satisfies(databaseSemver, '>=14.0.0 <19.0.0')) {
this.logger.error(`Database Backup Failure: Unsupported PostgreSQL version: ${databaseVersion}`);
return JobStatus.Failed;
}
this.logger.log(`Database Backup Starting. Database Version: ${databaseMajorVersion}`);
try {
await new Promise<void>((resolve, reject) => {
const pgdump = this.processRepository.spawn(
`/usr/lib/postgresql/${databaseMajorVersion}/bin/pg_dumpall`,
databaseParams,
{
env: {
PATH: process.env.PATH,
PGPASSWORD: isUrlConnection ? new URL(connectionUrl).password : config.password,
},
},
);
// NOTE: `--rsyncable` is only supported in GNU gzip
const gzip = this.processRepository.spawn(`gzip`, ['--rsyncable']);
pgdump.stdout.pipe(gzip.stdin);
const fileStream = this.storageRepository.createWriteStream(backupFilePath);
gzip.stdout.pipe(fileStream);
pgdump.on('error', (err) => {
this.logger.error(`Backup failed with error: ${err}`);
reject(err);
});
gzip.on('error', (err) => {
this.logger.error(`Gzip failed with error: ${err}`);
reject(err);
});
let pgdumpLogs = '';
let gzipLogs = '';
pgdump.stderr.on('data', (data) => (pgdumpLogs += data));
gzip.stderr.on('data', (data) => (gzipLogs += data));
pgdump.on('exit', (code) => {
if (code !== 0) {
this.logger.error(`Backup failed with code ${code}`);
reject(`Backup failed with code ${code}`);
this.logger.error(pgdumpLogs);
return;
}
if (pgdumpLogs) {
this.logger.debug(`pgdump_all logs\n${pgdumpLogs}`);
}
});
gzip.on('exit', (code) => {
if (code !== 0) {
this.logger.error(`Gzip failed with code ${code}`);
reject(`Gzip failed with code ${code}`);
this.logger.error(gzipLogs);
return;
}
if (pgdump.exitCode !== 0) {
this.logger.error(`Gzip exited with code 0 but pgdump exited with ${pgdump.exitCode}`);
return;
}
resolve();
});
});
await this.storageRepository.rename(backupFilePath, backupFilePath.replace('.tmp', ''));
await createDatabaseBackup(this.backupRepos);
} catch (error) {
this.logger.error(`Database Backup Failure: ${error}`);
await this.storageRepository
.unlink(backupFilePath)
.catch((error) => this.logger.error(`Failed to delete failed backup file: ${error}`));
if (error instanceof UnsupportedPostgresError) {
return JobStatus.Failed;
}
throw error;
}
this.logger.log(`Database Backup Success`);
await this.cleanupDatabaseBackups();
return JobStatus.Success;
}
private get backupRepos() {
return {
logger: this.logger,
storage: this.storageRepository,
config: this.configRepository,
process: this.processRepository,
database: this.databaseRepository,
};
}
}

View File

@@ -1,5 +1,5 @@
import { jwtVerify } from 'jose';
import { SystemMetadataKey } from 'src/enum';
import { MaintenanceAction, SystemMetadataKey } from 'src/enum';
import { CliService } from 'src/services/cli.service';
import { factory } from 'test/small.factory';
import { newTestService, ServiceMocks } from 'test/utils';
@@ -95,7 +95,14 @@ describe(CliService.name, () => {
});
it('should disable maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
mocks.systemMetadata.get.mockResolvedValue({
isMaintenanceMode: true,
secret: 'secret',
action: {
action: MaintenanceAction.Start,
},
});
await expect(sut.disableMaintenanceMode()).resolves.toEqual({
alreadyDisabled: false,
});
@@ -109,7 +116,14 @@ describe(CliService.name, () => {
describe('enableMaintenanceMode', () => {
it('should not do anything if in maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
mocks.systemMetadata.get.mockResolvedValue({
isMaintenanceMode: true,
secret: 'secret',
action: {
action: MaintenanceAction.Start,
},
});
await expect(sut.enableMaintenanceMode()).resolves.toEqual(
expect.objectContaining({
alreadyEnabled: true,
@@ -133,13 +147,22 @@ describe(CliService.name, () => {
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret: expect.stringMatching(/^\w{128}$/),
action: {
action: 'start',
},
});
});
const RE_LOGIN_URL = /https:\/\/my.immich.app\/maintenance\?token=([A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*)/;
it('should return a valid login URL', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
mocks.systemMetadata.get.mockResolvedValue({
isMaintenanceMode: true,
secret: 'secret',
action: {
action: MaintenanceAction.Start,
},
});
const result = await sut.enableMaintenanceMode();

View File

@@ -3,7 +3,7 @@ import { isAbsolute } from 'node:path';
import { SALT_ROUNDS } from 'src/constants';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';
import { SystemMetadataKey } from 'src/enum';
import { MaintenanceAction, SystemMetadataKey } from 'src/enum';
import { BaseService } from 'src/services/base.service';
import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance';
import { getExternalDomain } from 'src/utils/misc';
@@ -86,6 +86,9 @@ export class CliService extends BaseService {
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret,
action: {
action: MaintenanceAction.Start,
},
});
await this.appRepository.sendOneShotAppRestart({

View File

@@ -0,0 +1,83 @@
import { BadRequestException } from '@nestjs/common';
import { DateTime } from 'luxon';
import { StorageCore } from 'src/cores/storage.core';
import { StorageFolder } from 'src/enum';
import { DatabaseBackupService } from 'src/services/database-backup.service';
import { MaintenanceService } from 'src/services/maintenance.service';
import { newTestService, ServiceMocks } from 'test/utils';
describe(MaintenanceService.name, () => {
let sut: DatabaseBackupService;
let mocks: ServiceMocks;
beforeEach(() => {
({ sut, mocks } = newTestService(DatabaseBackupService));
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('listBackups', () => {
it('should give us all backups', async () => {
mocks.storage.readdir.mockResolvedValue([
`immich-db-backup-${DateTime.fromISO('2025-07-25T11:02:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
`immich-db-backup-${DateTime.fromISO('2025-07-27T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
'immich-db-backup-1753789649000.sql.gz',
`immich-db-backup-${DateTime.fromISO('2025-07-29T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
]);
mocks.storage.stat.mockResolvedValue({ size: 1024 } as any);
await expect(sut.listBackups()).resolves.toMatchObject({
backups: [
{ filename: 'immich-db-backup-20250729T110116-v1.234.5-pg14.5.sql.gz', filesize: 1024 },
{ filename: 'immich-db-backup-20250727T110116-v1.234.5-pg14.5.sql.gz', filesize: 1024 },
{ filename: 'immich-db-backup-1753789649000.sql.gz', filesize: 1024 },
],
});
});
});
describe('deleteBackup', () => {
it('should reject invalid file names', async () => {
await expect(sut.deleteBackup(['filename'])).rejects.toThrowError(
new BadRequestException('Invalid backup name!'),
);
});
it('should unlink the target file', async () => {
await sut.deleteBackup(['filename.sql']);
expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
expect(mocks.storage.unlink).toHaveBeenCalledWith(
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/filename.sql`,
);
});
});
describe('uploadBackup', () => {
it('should reject invalid file names', async () => {
await expect(sut.uploadBackup({ originalname: 'invalid backup' } as never)).rejects.toThrowError(
new BadRequestException('Invalid backup name!'),
);
});
it('should write file', async () => {
await sut.uploadBackup({ originalname: 'path.sql.gz', buffer: 'buffer' } as never);
expect(mocks.storage.createOrOverwriteFile).toBeCalledWith('/data/backups/uploaded-path.sql.gz', 'buffer');
});
});
describe('downloadBackup', () => {
it('should reject invalid file names', () => {
expect(() => sut.downloadBackup('invalid backup')).toThrowError(new BadRequestException('Invalid backup name!'));
});
it('should get backup path', () => {
expect(sut.downloadBackup('hello.sql.gz')).toEqual(
expect.objectContaining({
path: '/data/backups/hello.sql.gz',
}),
);
});
});
});

View File

@@ -0,0 +1,43 @@
import { Injectable } from '@nestjs/common';
import { DatabaseBackupListResponseDto } from 'src/dtos/database-backup.dto';
import { BaseService } from 'src/services/base.service';
import {
deleteDatabaseBackup,
downloadDatabaseBackup,
listDatabaseBackups,
uploadDatabaseBackup,
} from 'src/utils/database-backups';
import { ImmichFileResponse } from 'src/utils/file';
/**
* This service is available outside of maintenance mode to manage maintenance mode
*/
@Injectable()
export class DatabaseBackupService extends BaseService {
async listBackups(): Promise<DatabaseBackupListResponseDto> {
const backups = await listDatabaseBackups(this.backupRepos);
return { backups };
}
deleteBackup(files: string[]): Promise<void> {
return deleteDatabaseBackup(this.backupRepos, files);
}
async uploadBackup(file: Express.Multer.File): Promise<void> {
return uploadDatabaseBackup(this.backupRepos, file);
}
downloadBackup(fileName: string): ImmichFileResponse {
return downloadDatabaseBackup(fileName);
}
private get backupRepos() {
return {
logger: this.logger,
storage: this.storageRepository,
config: this.configRepository,
process: this.processRepository,
database: this.databaseRepository,
};
}
}

View File

@@ -9,6 +9,7 @@ import { AuthAdminService } from 'src/services/auth-admin.service';
import { AuthService } from 'src/services/auth.service';
import { BackupService } from 'src/services/backup.service';
import { CliService } from 'src/services/cli.service';
import { DatabaseBackupService } from 'src/services/database-backup.service';
import { DatabaseService } from 'src/services/database.service';
import { DownloadService } from 'src/services/download.service';
import { DuplicateService } from 'src/services/duplicate.service';
@@ -60,6 +61,7 @@ export const services = [
AuthAdminService,
BackupService,
CliService,
DatabaseBackupService,
DatabaseService,
DownloadService,
DuplicateService,

View File

@@ -136,7 +136,29 @@ export class JobService extends BaseService {
const asset = await this.assetRepository.getById(item.data.id);
if (asset) {
this.websocketRepository.clientSend('AssetEditReadyV1', asset.ownerId, { assetId: item.data.id });
this.websocketRepository.clientSend('AssetEditReadyV1', asset.ownerId, {
asset: {
id: asset.id,
ownerId: asset.ownerId,
originalFileName: asset.originalFileName,
thumbhash: asset.thumbhash ? hexOrBufferToBase64(asset.thumbhash) : null,
checksum: hexOrBufferToBase64(asset.checksum),
fileCreatedAt: asset.fileCreatedAt,
fileModifiedAt: asset.fileModifiedAt,
localDateTime: asset.localDateTime,
duration: asset.duration,
type: asset.type,
deletedAt: asset.deletedAt,
isFavorite: asset.isFavorite,
visibility: asset.visibility,
livePhotoVideoId: asset.livePhotoVideoId,
stackId: asset.stackId,
libraryId: asset.libraryId,
width: asset.width,
height: asset.height,
isEdited: asset.isEdited,
},
});
}
break;
@@ -189,6 +211,7 @@ export class JobService extends BaseService {
libraryId: asset.libraryId,
width: asset.width,
height: asset.height,
isEdited: asset.isEdited,
},
exif: {
assetId: exif.assetId,

View File

@@ -1,4 +1,4 @@
import { SystemMetadataKey } from 'src/enum';
import { MaintenanceAction, SystemMetadataKey } from 'src/enum';
import { MaintenanceService } from 'src/services/maintenance.service';
import { newTestService, ServiceMocks } from 'test/utils';
@@ -36,28 +36,96 @@ describe(MaintenanceService.name, () => {
});
it('should return true if enabled', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: '' });
mocks.systemMetadata.get.mockResolvedValue({
isMaintenanceMode: true,
secret: '',
action: { action: MaintenanceAction.Start },
});
await expect(sut.getMaintenanceMode()).resolves.toEqual({
isMaintenanceMode: true,
secret: '',
action: {
action: 'start',
},
});
expect(mocks.systemMetadata.get).toHaveBeenCalled();
});
});
describe('integrityCheck', () => {
it('generate integrity report', async () => {
mocks.storage.readdir.mockResolvedValue(['.immich', 'file1', 'file2']);
mocks.storage.readFile.mockResolvedValue(undefined as never);
mocks.storage.overwriteFile.mockRejectedValue(undefined as never);
await expect(sut.detectPriorInstall()).resolves.toMatchInlineSnapshot(`
{
"storage": [
{
"files": 2,
"folder": "encoded-video",
"readable": true,
"writable": false,
},
{
"files": 2,
"folder": "library",
"readable": true,
"writable": false,
},
{
"files": 2,
"folder": "upload",
"readable": true,
"writable": false,
},
{
"files": 2,
"folder": "profile",
"readable": true,
"writable": false,
},
{
"files": 2,
"folder": "thumbs",
"readable": true,
"writable": false,
},
{
"files": 2,
"folder": "backups",
"readable": true,
"writable": false,
},
],
}
`);
});
});
describe('startMaintenance', () => {
it('should set maintenance mode and return a secret', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(sut.startMaintenance('admin')).resolves.toMatchObject({
await expect(
sut.startMaintenance(
{
action: MaintenanceAction.Start,
},
'admin',
),
).resolves.toMatchObject({
jwt: expect.any(String),
});
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret: expect.stringMatching(/^\w{128}$/),
action: {
action: 'start',
},
});
expect(mocks.event.emit).toHaveBeenCalledWith('AppRestart', {
@@ -78,7 +146,13 @@ describe(MaintenanceService.name, () => {
});
it('should generate a login url with JWT', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
mocks.systemMetadata.get.mockResolvedValue({
isMaintenanceMode: true,
secret: 'secret',
action: {
action: MaintenanceAction.Start,
},
});
await expect(
sut.createLoginUrl({

View File

@@ -1,11 +1,21 @@
import { Injectable } from '@nestjs/common';
import { BadRequestException, Injectable } from '@nestjs/common';
import { OnEvent } from 'src/decorators';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { SystemMetadataKey } from 'src/enum';
import {
MaintenanceAuthDto,
MaintenanceDetectInstallResponseDto,
MaintenanceStatusResponseDto,
SetMaintenanceModeDto,
} from 'src/dtos/maintenance.dto';
import { MaintenanceAction, SystemMetadataKey } from 'src/enum';
import { ArgOf } from 'src/repositories/event.repository';
import { BaseService } from 'src/services/base.service';
import { MaintenanceModeState } from 'src/types';
import { createMaintenanceLoginUrl, generateMaintenanceSecret, signMaintenanceJwt } from 'src/utils/maintenance';
import {
createMaintenanceLoginUrl,
detectPriorInstall,
generateMaintenanceSecret,
signMaintenanceJwt,
} from 'src/utils/maintenance';
import { getExternalDomain } from 'src/utils/misc';
/**
@@ -19,9 +29,25 @@ export class MaintenanceService extends BaseService {
.then((state) => state ?? { isMaintenanceMode: false });
}
async startMaintenance(username: string): Promise<{ jwt: string }> {
getMaintenanceStatus(): MaintenanceStatusResponseDto {
return {
active: false,
action: MaintenanceAction.End,
};
}
detectPriorInstall(): Promise<MaintenanceDetectInstallResponseDto> {
return detectPriorInstall(this.storageRepository);
}
async startMaintenance(action: SetMaintenanceModeDto, username: string): Promise<{ jwt: string }> {
const secret = generateMaintenanceSecret();
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, { isMaintenanceMode: true, secret });
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret,
action,
});
await this.eventRepository.emit('AppRestart', { isMaintenanceMode: true });
return {
@@ -31,6 +57,20 @@ export class MaintenanceService extends BaseService {
};
}
async startRestoreFlow(): Promise<{ jwt: string }> {
const adminUser = await this.userRepository.getAdmin();
if (adminUser) {
throw new BadRequestException('The server already has an admin');
}
return this.startMaintenance(
{
action: MaintenanceAction.SelectDatabaseRestore,
},
'admin',
);
}
@OnEvent({ name: 'AppRestart', server: true })
onRestart(event: ArgOf<'AppRestart'>, ack?: (ok: 'ok') => void): void {
this.logger.log(`Restarting due to event... ${JSON.stringify(event)}`);

View File

@@ -241,21 +241,21 @@ describe(MediaService.name, () => {
await expect(sut.handleAssetMigration({ id: assetStub.image.id })).resolves.toBe(JobStatus.Success);
expect(mocks.move.create).toHaveBeenCalledWith({
entityId: assetStub.image.id,
pathType: AssetPathType.FullSize,
pathType: AssetFileType.FullSize,
oldPath: '/uploads/user-id/fullsize/path.webp',
newPath: expect.stringContaining('/data/thumbs/user-id/as/se/asset-id-fullsize.jpeg'),
newPath: expect.stringContaining('/data/thumbs/user-id/as/se/asset-id_fullsize.jpeg'),
});
expect(mocks.move.create).toHaveBeenCalledWith({
entityId: assetStub.image.id,
pathType: AssetPathType.Preview,
pathType: AssetFileType.Preview,
oldPath: '/uploads/user-id/thumbs/path.jpg',
newPath: expect.stringContaining('/data/thumbs/user-id/as/se/asset-id-preview.jpeg'),
newPath: expect.stringContaining('/data/thumbs/user-id/as/se/asset-id_preview.jpeg'),
});
expect(mocks.move.create).toHaveBeenCalledWith({
entityId: assetStub.image.id,
pathType: AssetPathType.Thumbnail,
pathType: AssetFileType.Thumbnail,
oldPath: '/uploads/user-id/webp/path.ext',
newPath: expect.stringContaining('/data/thumbs/user-id/as/se/asset-id-thumbnail.webp'),
newPath: expect.stringContaining('/data/thumbs/user-id/as/se/asset-id_thumbnail.webp'),
});
expect(mocks.move.create).toHaveBeenCalledTimes(3);
});
@@ -385,11 +385,13 @@ describe(MediaService.name, () => {
assetId: 'asset-id',
type: AssetFileType.Preview,
path: expect.any(String),
isEdited: false,
},
{
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: expect.any(String),
isEdited: false,
},
]);
expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'asset-id', thumbhash: thumbhashBuffer });
@@ -421,11 +423,13 @@ describe(MediaService.name, () => {
assetId: 'asset-id',
type: AssetFileType.Preview,
path: expect.any(String),
isEdited: false,
},
{
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: expect.any(String),
isEdited: false,
},
]);
});
@@ -456,11 +460,13 @@ describe(MediaService.name, () => {
assetId: 'asset-id',
type: AssetFileType.Preview,
path: expect.any(String),
isEdited: false,
},
{
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: expect.any(String),
isEdited: false,
},
]);
});
@@ -548,8 +554,8 @@ describe(MediaService.name, () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image);
const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
mocks.media.generateThumbhash.mockResolvedValue(thumbhashBuffer);
const previewPath = `/data/thumbs/user-id/as/se/asset-id-preview.${format}`;
const thumbnailPath = `/data/thumbs/user-id/as/se/asset-id-thumbnail.webp`;
const previewPath = `/data/thumbs/user-id/as/se/asset-id_preview.${format}`;
const thumbnailPath = `/data/thumbs/user-id/as/se/asset-id_thumbnail.webp`;
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
@@ -595,8 +601,8 @@ describe(MediaService.name, () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image);
const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
mocks.media.generateThumbhash.mockResolvedValue(thumbhashBuffer);
const previewPath = expect.stringContaining(`/data/thumbs/user-id/as/se/asset-id-preview.jpeg`);
const thumbnailPath = expect.stringContaining(`/data/thumbs/user-id/as/se/asset-id-thumbnail.${format}`);
const previewPath = expect.stringContaining(`/data/thumbs/user-id/as/se/asset-id_preview.jpeg`);
const thumbnailPath = expect.stringContaining(`/data/thumbs/user-id/as/se/asset-id_thumbnail.${format}`);
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
@@ -1026,9 +1032,9 @@ describe(MediaService.name, () => {
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining({ type: AssetFileType.FullSizeEdited }),
expect.objectContaining({ type: AssetFileType.PreviewEdited }),
expect.objectContaining({ type: AssetFileType.ThumbnailEdited }),
expect.objectContaining({ type: AssetFileType.FullSize, isEdited: true }),
expect.objectContaining({ type: AssetFileType.Preview, isEdited: true }),
expect.objectContaining({ type: AssetFileType.Thumbnail, isEdited: true }),
]),
);
});
@@ -1098,17 +1104,17 @@ describe(MediaService.name, () => {
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.anything(),
expect.stringContaining('edited_preview.jpeg'),
expect.stringContaining('preview_edited.jpeg'),
);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.anything(),
expect.stringContaining('edited_thumbnail.webp'),
expect.stringContaining('thumbnail_edited.webp'),
);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.anything(),
expect.stringContaining('edited_fullsize.jpeg'),
expect.stringContaining('fullsize_edited.jpeg'),
);
});
@@ -3254,13 +3260,13 @@ describe(MediaService.name, () => {
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg' },
{ type: AssetFileType.Thumbnail, newPath: '/new/thumbnail.jpg' },
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg', isEdited: false },
{ type: AssetFileType.Thumbnail, newPath: '/new/thumbnail.jpg', isEdited: false },
]);
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview },
{ assetId: 'asset-id', path: '/new/thumbnail.jpg', type: AssetFileType.Thumbnail },
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview, isEdited: false },
{ assetId: 'asset-id', path: '/new/thumbnail.jpg', type: AssetFileType.Thumbnail, isEdited: false },
]);
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).not.toHaveBeenCalled();
@@ -3270,19 +3276,31 @@ describe(MediaService.name, () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
{
id: 'file-1',
assetId: 'asset-id',
type: AssetFileType.Preview,
path: '/old/preview.jpg',
isEdited: false,
},
{
id: 'file-2',
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: '/old/thumbnail.jpg',
isEdited: false,
},
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg' },
{ type: AssetFileType.Thumbnail, newPath: '/new/thumbnail.jpg' },
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg', isEdited: false },
{ type: AssetFileType.Thumbnail, newPath: '/new/thumbnail.jpg', isEdited: false },
]);
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview },
{ assetId: 'asset-id', path: '/new/thumbnail.jpg', type: AssetFileType.Thumbnail },
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview, isEdited: false },
{ assetId: 'asset-id', path: '/new/thumbnail.jpg', type: AssetFileType.Thumbnail, isEdited: false },
]);
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).toHaveBeenCalledWith({
@@ -3295,17 +3313,38 @@ describe(MediaService.name, () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
{
id: 'file-1',
assetId: 'asset-id',
type: AssetFileType.Preview,
path: '/old/preview.jpg',
isEdited: false,
},
{
id: 'file-2',
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: '/old/thumbnail.jpg',
isEdited: false,
},
],
};
await sut['syncFiles'](asset, [{ type: AssetFileType.Preview }, { type: AssetFileType.Thumbnail }]);
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, isEdited: false },
{ type: AssetFileType.Thumbnail, isEdited: false },
]);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg', isEdited: false },
{
id: 'file-2',
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: '/old/thumbnail.jpg',
isEdited: false,
},
]);
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
@@ -3317,14 +3356,26 @@ describe(MediaService.name, () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/same/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/same/thumbnail.jpg' },
{
id: 'file-1',
assetId: 'asset-id',
type: AssetFileType.Preview,
path: '/same/preview.jpg',
isEdited: false,
},
{
id: 'file-2',
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: '/same/thumbnail.jpg',
isEdited: false,
},
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/same/preview.jpg' },
{ type: AssetFileType.Thumbnail, newPath: '/same/thumbnail.jpg' },
{ type: AssetFileType.Preview, newPath: '/same/preview.jpg', isEdited: false },
{ type: AssetFileType.Thumbnail, newPath: '/same/thumbnail.jpg', isEdited: false },
]);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
@@ -3336,23 +3387,41 @@ describe(MediaService.name, () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
{
id: 'file-1',
assetId: 'asset-id',
type: AssetFileType.Preview,
path: '/old/preview.jpg',
isEdited: false,
},
{
id: 'file-2',
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: '/old/thumbnail.jpg',
isEdited: false,
},
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg' }, // replace
{ type: AssetFileType.Thumbnail }, // delete
{ type: AssetFileType.FullSize, newPath: '/new/fullsize.jpg' }, // new
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg', isEdited: false }, // replace
{ type: AssetFileType.Thumbnail, isEdited: false }, // delete
{ type: AssetFileType.FullSize, newPath: '/new/fullsize.jpg', isEdited: false }, // new
]);
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview },
{ assetId: 'asset-id', path: '/new/fullsize.jpg', type: AssetFileType.FullSize },
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview, isEdited: false },
{ assetId: 'asset-id', path: '/new/fullsize.jpg', type: AssetFileType.FullSize, isEdited: false },
]);
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
{
id: 'file-2',
assetId: 'asset-id',
type: AssetFileType.Thumbnail,
path: '/old/thumbnail.jpg',
isEdited: false,
},
]);
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
@@ -3376,11 +3445,19 @@ describe(MediaService.name, () => {
it('should delete non-existent file types when newPath is not provided', async () => {
const asset = {
id: 'asset-id',
files: [{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' }],
files: [
{
id: 'file-1',
assetId: 'asset-id',
type: AssetFileType.Preview,
path: '/old/preview.jpg',
isEdited: false,
},
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Thumbnail }, // file doesn't exist, newPath not provided
{ type: AssetFileType.Thumbnail, isEdited: false }, // file doesn't exist, newPath not provided
]);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();

View File

@@ -8,7 +8,6 @@ import { AssetEditAction, CropParameters } from 'src/dtos/editing.dto';
import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
import {
AssetFileType,
AssetPathType,
AssetType,
AssetVisibility,
AudioCodec,
@@ -50,6 +49,7 @@ interface UpsertFileOptions {
assetId: string;
type: AssetFileType;
path: string;
isEdited: boolean;
}
type ThumbnailAsset = NonNullable<Awaited<ReturnType<AssetJobRepository['getForGenerateThumbnailJob']>>>;
@@ -160,9 +160,9 @@ export class MediaService extends BaseService {
return JobStatus.Failed;
}
await this.storageCore.moveAssetImage(asset, AssetPathType.FullSize, image.fullsize.format);
await this.storageCore.moveAssetImage(asset, AssetPathType.Preview, image.preview.format);
await this.storageCore.moveAssetImage(asset, AssetPathType.Thumbnail, image.thumbnail.format);
await this.storageCore.moveAssetImage(asset, AssetFileType.FullSize, image.fullsize.format);
await this.storageCore.moveAssetImage(asset, AssetFileType.Preview, image.preview.format);
await this.storageCore.moveAssetImage(asset, AssetFileType.Thumbnail, image.thumbnail.format);
await this.storageCore.moveAssetVideo(asset);
return JobStatus.Success;
@@ -236,9 +236,9 @@ export class MediaService extends BaseService {
}
await this.syncFiles(asset, [
{ type: AssetFileType.Preview, newPath: generated.previewPath },
{ type: AssetFileType.Thumbnail, newPath: generated.thumbnailPath },
{ type: AssetFileType.FullSize, newPath: generated.fullsizePath },
{ type: AssetFileType.Preview, newPath: generated.previewPath, isEdited: false },
{ type: AssetFileType.Thumbnail, newPath: generated.thumbnailPath, isEdited: false },
{ type: AssetFileType.FullSize, newPath: generated.fullsizePath, isEdited: false },
]);
const editiedGenerated = await this.generateEditedThumbnails(asset);
@@ -307,16 +307,16 @@ export class MediaService extends BaseService {
private async generateImageThumbnails(asset: ThumbnailAsset, useEdits: boolean = false) {
const { image } = await this.getConfig({ withCache: true });
const previewPath = StorageCore.getImagePath(
asset,
useEdits ? AssetPathType.EditedPreview : AssetPathType.Preview,
image.preview.format,
);
const thumbnailPath = StorageCore.getImagePath(
asset,
useEdits ? AssetPathType.EditedThumbnail : AssetPathType.Thumbnail,
image.thumbnail.format,
);
const previewPath = StorageCore.getImagePath(asset, {
fileType: AssetFileType.Preview,
isEdited: useEdits,
format: image.preview.format,
});
const thumbnailPath = StorageCore.getImagePath(asset, {
fileType: AssetFileType.Thumbnail,
isEdited: useEdits,
format: image.thumbnail.format,
});
this.storageCore.ensureFolders(previewPath);
// Handle embedded preview extraction for RAW files
@@ -343,11 +343,11 @@ export class MediaService extends BaseService {
if (convertFullsize) {
// convert a new fullsize image from the same source as the thumbnail
fullsizePath = StorageCore.getImagePath(
asset,
useEdits ? AssetPathType.EditedFullSize : AssetPathType.FullSize,
image.fullsize.format,
);
fullsizePath = StorageCore.getImagePath(asset, {
fileType: AssetFileType.FullSize,
isEdited: useEdits,
format: image.fullsize.format,
});
const fullsizeOptions = {
format: image.fullsize.format,
quality: image.fullsize.quality,
@@ -355,7 +355,11 @@ export class MediaService extends BaseService {
};
promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath));
} else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) {
fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, extracted.format);
fullsizePath = StorageCore.getImagePath(asset, {
fileType: AssetFileType.FullSize,
format: extracted.format,
isEdited: false,
});
this.storageCore.ensureFolders(fullsizePath);
// Write the buffer to disk with essential EXIF data
@@ -489,8 +493,16 @@ export class MediaService extends BaseService {
private async generateVideoThumbnails(asset: ThumbnailPathEntity & { originalPath: string }) {
const { image, ffmpeg } = await this.getConfig({ withCache: true });
const previewPath = StorageCore.getImagePath(asset, AssetPathType.Preview, image.preview.format);
const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.Thumbnail, image.thumbnail.format);
const previewPath = StorageCore.getImagePath(asset, {
fileType: AssetFileType.Preview,
format: image.preview.format,
isEdited: false,
});
const thumbnailPath = StorageCore.getImagePath(asset, {
fileType: AssetFileType.Thumbnail,
format: image.thumbnail.format,
isEdited: false,
});
this.storageCore.ensureFolders(previewPath);
const { format, audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
@@ -779,18 +791,18 @@ export class MediaService extends BaseService {
private async syncFiles(
asset: { id: string; files: AssetFile[] },
files: { type: AssetFileType; newPath?: string }[],
files: { type: AssetFileType; newPath?: string; isEdited: boolean }[],
) {
const toUpsert: UpsertFileOptions[] = [];
const pathsToDelete: string[] = [];
const toDelete: AssetFile[] = [];
for (const { type, newPath } of files) {
const existingFile = asset.files.find((file) => file.type === type);
for (const { type, newPath, isEdited } of files) {
const existingFile = asset.files.find((file) => file.type === type && file.isEdited === isEdited);
// upsert new file path
if (newPath && existingFile?.path !== newPath) {
toUpsert.push({ assetId: asset.id, path: newPath, type });
toUpsert.push({ assetId: asset.id, path: newPath, type, isEdited });
// delete old file from disk
if (existingFile) {
@@ -829,9 +841,9 @@ export class MediaService extends BaseService {
const generated = asset.edits.length > 0 ? await this.generateImageThumbnails(asset, true) : undefined;
await this.syncFiles(asset, [
{ type: AssetFileType.PreviewEdited, newPath: generated?.previewPath },
{ type: AssetFileType.ThumbnailEdited, newPath: generated?.thumbnailPath },
{ type: AssetFileType.FullSizeEdited, newPath: generated?.fullsizePath },
{ type: AssetFileType.Preview, newPath: generated?.previewPath, isEdited: true },
{ type: AssetFileType.Thumbnail, newPath: generated?.thumbnailPath, isEdited: true },
{ type: AssetFileType.FullSize, newPath: generated?.fullsizePath, isEdited: true },
]);
const crop = asset.edits.find((e) => e.action === AssetEditAction.Crop);

View File

@@ -35,7 +35,7 @@ const forSidecarJob = (
asset: {
id?: string;
originalPath?: string;
files?: { id: string; type: AssetFileType; path: string }[];
files?: { id: string; type: AssetFileType; path: string; isEdited: boolean }[];
} = {},
) => {
return {
@@ -387,6 +387,7 @@ describe(MetadataService.name, () => {
it('should extract tags from TagsList', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent'] } } as any);
mockReadTags({ TagsList: ['Parent'] });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -397,6 +398,7 @@ describe(MetadataService.name, () => {
it('should extract hierarchy from TagsList', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent/Child'] } } as any);
mockReadTags({ TagsList: ['Parent/Child'] });
mocks.tag.upsertValue.mockResolvedValueOnce(tagStub.parentUpsert);
mocks.tag.upsertValue.mockResolvedValueOnce(tagStub.childUpsert);
@@ -417,6 +419,7 @@ describe(MetadataService.name, () => {
it('should extract tags from Keywords as a string', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent'] } } as any);
mockReadTags({ Keywords: 'Parent' });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -427,6 +430,7 @@ describe(MetadataService.name, () => {
it('should extract tags from Keywords as a list', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent'] } } as any);
mockReadTags({ Keywords: ['Parent'] });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -437,6 +441,7 @@ describe(MetadataService.name, () => {
it('should extract tags from Keywords as a list with a number', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent', '2024'] } } as any);
mockReadTags({ Keywords: ['Parent', 2024] });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -448,6 +453,7 @@ describe(MetadataService.name, () => {
it('should extract hierarchal tags from Keywords', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent/Child'] } } as any);
mockReadTags({ Keywords: 'Parent/Child' });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -467,6 +473,7 @@ describe(MetadataService.name, () => {
it('should ignore Keywords when TagsList is present', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent/Child', 'Child'] } } as any);
mockReadTags({ Keywords: 'Child', TagsList: ['Parent/Child'] });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -486,6 +493,7 @@ describe(MetadataService.name, () => {
it('should extract hierarchy from HierarchicalSubject', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent/Child', 'TagA'] } } as any);
mockReadTags({ HierarchicalSubject: ['Parent|Child', 'TagA'] });
mocks.tag.upsertValue.mockResolvedValueOnce(tagStub.parentUpsert);
mocks.tag.upsertValue.mockResolvedValueOnce(tagStub.childUpsert);
@@ -507,6 +515,7 @@ describe(MetadataService.name, () => {
it('should extract tags from HierarchicalSubject as a list with a number', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent', '2024'] } } as any);
mockReadTags({ HierarchicalSubject: ['Parent', 2024] });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -518,6 +527,7 @@ describe(MetadataService.name, () => {
it('should extract ignore / characters in a HierarchicalSubject tag', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.image);
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Mom|Dad'] } } as any);
mockReadTags({ HierarchicalSubject: ['Mom/Dad'] });
mocks.tag.upsertValue.mockResolvedValueOnce(tagStub.parentUpsert);
@@ -532,6 +542,7 @@ describe(MetadataService.name, () => {
it('should ignore HierarchicalSubject when TagsList is present', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.image);
mocks.asset.getById.mockResolvedValue({ exifInfo: { tags: ['Parent/Child', 'Parent2/Child2'] } } as any);
mockReadTags({ HierarchicalSubject: ['Parent2|Child2'], TagsList: ['Parent/Child'] });
mocks.tag.upsertValue.mockResolvedValue(tagStub.parentUpsert);
@@ -896,6 +907,7 @@ describe(MetadataService.name, () => {
ProfileDescription: 'extensive description',
ProjectionType: 'equirectangular',
tz: 'UTC-11:30',
TagsList: ['parent/child'],
Rating: 3,
};
@@ -935,6 +947,7 @@ describe(MetadataService.name, () => {
country: null,
state: null,
city: null,
tags: ['parent/child'],
},
{ lockedPropertiesBehavior: 'skip' },
);
@@ -1084,6 +1097,7 @@ describe(MetadataService.name, () => {
id: 'some-id',
type: AssetFileType.Sidecar,
path: '/path/to/something',
isEdited: false,
},
],
});
@@ -1691,7 +1705,7 @@ describe(MetadataService.name, () => {
it('should unset sidecar path if file no longer exist', async () => {
const asset = forSidecarJob({
originalPath: '/path/to/IMG_123.jpg',
files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar }],
files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar, isEdited: false }],
});
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
mocks.storage.checkFileExists.mockResolvedValue(false);
@@ -1704,7 +1718,7 @@ describe(MetadataService.name, () => {
it('should do nothing if the sidecar file still exists', async () => {
const asset = forSidecarJob({
originalPath: '/path/to/IMG_123.jpg',
files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar }],
files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar, isEdited: false }],
});
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);

View File

@@ -254,6 +254,8 @@ export class MetadataService extends BaseService {
}
}
const tags = this.getTagList(exifTags);
const exifData: Insertable<AssetExifTable> = {
assetId: asset.id,
@@ -296,6 +298,8 @@ export class MetadataService extends BaseService {
// grouping
livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null,
autoStackId: this.getAutoStackId(exifTags),
tags: tags.length > 0 ? tags : null,
};
const isSidewards = exifTags.Orientation && this.isOrientationSidewards(exifTags.Orientation);
@@ -316,9 +320,10 @@ export class MetadataService extends BaseService {
width: asset.width == null ? assetWidth : undefined,
height: asset.height == null ? assetHeight : undefined,
}),
this.applyTagList(asset, exifTags),
];
await this.applyTagList(asset);
if (this.isMotionPhoto(asset, exifTags)) {
promises.push(this.applyMotionPhotos(asset, exifTags, dates, stats));
}
@@ -405,35 +410,35 @@ export class MetadataService extends BaseService {
@OnEvent({ name: 'AssetTag' })
async handleTagAsset({ assetId }: ArgOf<'AssetTag'>) {
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id: assetId, tags: true } });
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id: assetId } });
}
@OnEvent({ name: 'AssetUntag' })
async handleUntagAsset({ assetId }: ArgOf<'AssetUntag'>) {
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id: assetId, tags: true } });
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id: assetId } });
}
@OnJob({ name: JobName.SidecarWrite, queue: QueueName.Sidecar })
async handleSidecarWrite(job: JobOf<JobName.SidecarWrite>): Promise<JobStatus> {
const { id, tags } = job;
const { id } = job;
const asset = await this.assetJobRepository.getForSidecarWriteJob(id);
if (!asset) {
return JobStatus.Failed;
}
const lockedProperties = await this.assetJobRepository.getLockedPropertiesForMetadataExtraction(id);
const tagsList = (asset.tags || []).map((tag) => tag.value);
const { sidecarFile } = getAssetFiles(asset.files);
const sidecarPath = sidecarFile?.path || `${asset.originalPath}.xmp`;
const { description, dateTimeOriginal, latitude, longitude, rating } = _.pick(
const { description, dateTimeOriginal, latitude, longitude, rating, tags } = _.pick(
{
description: asset.exifInfo.description,
dateTimeOriginal: asset.exifInfo.dateTimeOriginal,
latitude: asset.exifInfo.latitude,
longitude: asset.exifInfo.longitude,
rating: asset.exifInfo.rating,
tags: asset.exifInfo.tags,
},
lockedProperties,
);
@@ -446,7 +451,7 @@ export class MetadataService extends BaseService {
GPSLatitude: latitude,
GPSLongitude: longitude,
Rating: rating,
TagsList: tags ? tagsList : undefined,
TagsList: tags?.length ? tags : undefined,
},
_.isUndefined,
);
@@ -560,11 +565,14 @@ export class MetadataService extends BaseService {
return tags;
}
private async applyTagList(asset: { id: string; ownerId: string }, exifTags: ImmichTags) {
const tags = this.getTagList(exifTags);
const results = await upsertTags(this.tagRepository, { userId: asset.ownerId, tags });
private async applyTagList({ id, ownerId }: { id: string; ownerId: string }) {
const asset = await this.assetRepository.getById(id, { exifInfo: true });
const results = await upsertTags(this.tagRepository, {
userId: ownerId,
tags: asset?.exifInfo?.tags ?? [],
});
await this.tagRepository.replaceAssetTags(
asset.id,
id,
results.map((tag) => tag.id),
);
}

View File

@@ -372,7 +372,7 @@ describe(NotificationService.name, () => {
mocks.notification.create.mockResolvedValue(notificationStub.albumEvent);
mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' });
mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([
{ id: '1', type: AssetFileType.Thumbnail, path: 'path-to-thumb.jpg' },
{ id: '1', type: AssetFileType.Thumbnail, path: 'path-to-thumb.jpg', isEdited: false },
]);
await expect(sut.handleAlbumInvite({ id: '', recipientId: '' })).resolves.toBe(JobStatus.Success);
@@ -403,7 +403,7 @@ describe(NotificationService.name, () => {
mocks.systemMetadata.get.mockResolvedValue({ server: {} });
mocks.notification.create.mockResolvedValue(notificationStub.albumEvent);
mocks.email.renderEmail.mockResolvedValue({ html: '', text: '' });
mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([assetStub.image.files[2]]);
mocks.assetJob.getAlbumThumbnailFiles.mockResolvedValue([{ ...assetStub.image.files[2], isEdited: false }]);
await expect(sut.handleAlbumInvite({ id: '', recipientId: '' })).resolves.toBe(JobStatus.Success);
expect(mocks.assetJob.getAlbumThumbnailFiles).toHaveBeenCalledWith(

View File

@@ -117,7 +117,7 @@ export class SmartInfoService extends BaseService {
const newConfig = await this.getConfig({ withCache: true });
if (machineLearning.clip.modelName !== newConfig.machineLearning.clip.modelName) {
// Skip the job if the the model has changed since the embedding was generated.
// Skip the job if the model has changed since the embedding was generated.
return JobStatus.Skipped;
}

View File

@@ -240,11 +240,11 @@ export class StorageTemplateService extends BaseService {
assetInfo: { sizeInBytes: fileSizeInByte, checksum },
});
const sidecarPath = getAssetFile(asset.files, AssetFileType.Sidecar)?.path;
const sidecarPath = getAssetFile(asset.files, AssetFileType.Sidecar, { isEdited: false })?.path;
if (sidecarPath) {
await this.storageCore.moveFile({
entityId: id,
pathType: AssetPathType.Sidecar,
pathType: AssetFileType.Sidecar,
oldPath: sidecarPath,
newPath: `${newPath}.xmp`,
});

View File

@@ -191,6 +191,7 @@ describe(TagService.name, () => {
it('should upsert records', async () => {
mocks.access.tag.checkOwnerAccess.mockResolvedValue(new Set(['tag-1', 'tag-2']));
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.asset.getById.mockResolvedValue({ tags: [{ value: 'tag-1' }, { value: 'tag-2' }] } as any);
mocks.tag.upsertAssetIds.mockResolvedValue([
{ tagId: 'tag-1', assetId: 'asset-1' },
{ tagId: 'tag-1', assetId: 'asset-2' },
@@ -204,6 +205,18 @@ describe(TagService.name, () => {
).resolves.toEqual({
count: 6,
});
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-1', tags: ['tag-1', 'tag-2'] },
{ lockedPropertiesBehavior: 'append' },
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-2', tags: ['tag-1', 'tag-2'] },
{ lockedPropertiesBehavior: 'append' },
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-3', tags: ['tag-1', 'tag-2'] },
{ lockedPropertiesBehavior: 'append' },
);
expect(mocks.tag.upsertAssetIds).toHaveBeenCalledWith([
{ tagId: 'tag-1', assetId: 'asset-1' },
{ tagId: 'tag-1', assetId: 'asset-2' },
@@ -229,6 +242,7 @@ describe(TagService.name, () => {
mocks.tag.get.mockResolvedValue(tagStub.tag);
mocks.tag.getAssetIds.mockResolvedValue(new Set(['asset-1']));
mocks.tag.addAssetIds.mockResolvedValue();
mocks.asset.getById.mockResolvedValue({ tags: [{ value: 'tag-1' }] } as any);
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-2']));
await expect(
@@ -240,6 +254,14 @@ describe(TagService.name, () => {
{ id: 'asset-2', success: true },
]);
expect(mocks.asset.upsertExif).not.toHaveBeenCalledWith(
{ assetId: 'asset-1', tags: ['tag-1'] },
{ lockedPropertiesBehavior: 'append' },
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-2', tags: ['tag-1'] },
{ lockedPropertiesBehavior: 'append' },
);
expect(mocks.tag.getAssetIds).toHaveBeenCalledWith('tag-1', ['asset-1', 'asset-2']);
expect(mocks.tag.addAssetIds).toHaveBeenCalledWith('tag-1', ['asset-2']);
});

View File

@@ -90,6 +90,7 @@ export class TagService extends BaseService {
const results = await this.tagRepository.upsertAssetIds(items);
for (const assetId of new Set(results.map((item) => item.assetId))) {
await this.updateTags(assetId);
await this.eventRepository.emit('AssetTag', { assetId });
}
@@ -107,6 +108,7 @@ export class TagService extends BaseService {
for (const { id: assetId, success } of results) {
if (success) {
await this.updateTags(assetId);
await this.eventRepository.emit('AssetTag', { assetId });
}
}
@@ -125,6 +127,7 @@ export class TagService extends BaseService {
for (const { id: assetId, success } of results) {
if (success) {
await this.updateTags(assetId);
await this.eventRepository.emit('AssetUntag', { assetId });
}
}
@@ -145,4 +148,12 @@ export class TagService extends BaseService {
}
return tag;
}
private async updateTags(assetId: string) {
const asset = await this.assetRepository.getById(assetId, { tags: true });
await this.assetRepository.upsertExif(
{ assetId, tags: asset?.tags?.map(({ value }) => value) ?? [] },
{ lockedPropertiesBehavior: 'append' },
);
}
}