mirror of
https://github.com/immich-app/immich.git
synced 2026-02-15 13:28:24 +03:00
refactor(server): move database restores code into a service (#25918)
* fix(server): use provided database name/username for restore & ensure name is not mangled fixes #25633 Signed-off-by: izzy <me@insrt.uk> * chore: add db switch back but with comments Signed-off-by: izzy <me@insrt.uk> * refactor: no need to restore database since it's not technically possible chore: late fallback for username in parameter builder Signed-off-by: izzy <me@insrt.uk> * chore: type fix Signed-off-by: izzy <me@insrt.uk> * refactor: move db backup code into service * test: check SQL sent to psql * chore: remove todo Signed-off-by: izzy <me@insrt.uk> --------- Signed-off-by: izzy <me@insrt.uk>
This commit is contained in:
Paul Makles
@@ -1,270 +0,0 @@
|
||||
import { DateTime } from 'luxon';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import { defaults, SystemConfig } from 'src/config';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { ImmichWorker, JobStatus, StorageFolder } from 'src/enum';
|
||||
import { BackupService } from 'src/services/backup.service';
|
||||
import { systemConfigStub } from 'test/fixtures/system-config.stub';
|
||||
import { mockDuplex, mockSpawn, newTestService, ServiceMocks } from 'test/utils';
|
||||
import { describe } from 'vitest';
|
||||
|
||||
describe(BackupService.name, () => {
|
||||
let sut: BackupService;
|
||||
let mocks: ServiceMocks;
|
||||
|
||||
beforeEach(() => {
|
||||
({ sut, mocks } = newTestService(BackupService));
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('onBootstrapEvent', () => {
|
||||
it('should init cron job and handle config changes', async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(true);
|
||||
mocks.cron.create.mockResolvedValue();
|
||||
|
||||
await sut.onConfigInit({ newConfig: systemConfigStub.backupEnabled as SystemConfig });
|
||||
|
||||
expect(mocks.cron.create).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not initialize backup database cron job when lock is taken', async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(false);
|
||||
|
||||
await sut.onConfigInit({ newConfig: systemConfigStub.backupEnabled as SystemConfig });
|
||||
|
||||
expect(mocks.cron.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not initialise backup database job when running on microservices', async () => {
|
||||
mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices);
|
||||
await sut.onConfigInit({ newConfig: systemConfigStub.backupEnabled as SystemConfig });
|
||||
|
||||
expect(mocks.cron.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('onConfigUpdateEvent', () => {
|
||||
beforeEach(async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(true);
|
||||
mocks.cron.create.mockResolvedValue();
|
||||
|
||||
await sut.onConfigInit({ newConfig: defaults });
|
||||
});
|
||||
|
||||
it('should update cron job if backup is enabled', () => {
|
||||
mocks.cron.update.mockResolvedValue();
|
||||
|
||||
sut.onConfigUpdate({
|
||||
oldConfig: defaults,
|
||||
newConfig: {
|
||||
backup: {
|
||||
database: {
|
||||
enabled: true,
|
||||
cronExpression: '0 1 * * *',
|
||||
},
|
||||
},
|
||||
} as SystemConfig,
|
||||
});
|
||||
|
||||
expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'backupDatabase', expression: '0 1 * * *', start: true });
|
||||
expect(mocks.cron.update).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should do nothing if instance does not have the backup database lock', async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(false);
|
||||
await sut.onConfigInit({ newConfig: defaults });
|
||||
sut.onConfigUpdate({ newConfig: systemConfigStub.backupEnabled as SystemConfig, oldConfig: defaults });
|
||||
expect(mocks.cron.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupDatabaseBackups', () => {
|
||||
it('should do nothing if not reached keepLastAmount', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.readdir.mockResolvedValue(['immich-db-backup-1.sql.gz']);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should remove failed backup files', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
//`immich-db-backup-${DateTime.now().toFormat("yyyyLLdd'T'HHmmss")}-v${serverVersion.toString()}-pg${databaseVersion.split(' ')[0]}.sql.gz.tmp`,
|
||||
mocks.storage.readdir.mockResolvedValue([
|
||||
'immich-db-backup-123.sql.gz.tmp',
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-25T11:02:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-27T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-29T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
]);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-123.sql.gz.tmp`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250725T110216-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250729T110116-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
);
|
||||
});
|
||||
|
||||
it('should remove old backup files over keepLastAmount', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.readdir.mockResolvedValue(['immich-db-backup-1.sql.gz', 'immich-db-backup-2.sql.gz']);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-1.sql.gz`,
|
||||
);
|
||||
});
|
||||
|
||||
it('should remove old backup files over keepLastAmount and failed backups', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.readdir.mockResolvedValue([
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-25T11:02:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-27T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
|
||||
'immich-db-backup-1753789649000.sql.gz',
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-29T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
|
||||
]);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-1753789649000.sql.gz`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250725T110216-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250727T110116-v1.234.5-pg14.5.sql.gz`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleBackupDatabase', () => {
|
||||
beforeEach(() => {
|
||||
mocks.storage.readdir.mockResolvedValue([]);
|
||||
mocks.process.spawn.mockReturnValue(mockSpawn(0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex('command', 0, 'data', ''));
|
||||
mocks.storage.rename.mockResolvedValue();
|
||||
mocks.storage.unlink.mockResolvedValue();
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.createWriteStream.mockReturnValue(new PassThrough());
|
||||
});
|
||||
|
||||
it('should sanitize DB_URL (remove uselibpqcompat) before calling pg_dumpall', async () => {
|
||||
// create a service instance with a URL connection that includes libpqcompat
|
||||
const dbUrl = 'postgresql://postgres:pwd@host:5432/immich?sslmode=require&uselibpqcompat=true';
|
||||
const configMock = {
|
||||
getEnv: () => ({ database: { config: { connectionType: 'url', url: dbUrl }, skipMigrations: false } }),
|
||||
getWorker: () => ImmichWorker.Api,
|
||||
isDev: () => false,
|
||||
} as unknown as any;
|
||||
|
||||
({ sut, mocks } = newTestService(BackupService, { config: configMock }));
|
||||
|
||||
mocks.storage.readdir.mockResolvedValue([]);
|
||||
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex('command', 0, 'data', ''));
|
||||
mocks.storage.rename.mockResolvedValue();
|
||||
mocks.storage.unlink.mockResolvedValue();
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.createWriteStream.mockReturnValue(new PassThrough());
|
||||
mocks.database.getPostgresVersion.mockResolvedValue('14.10');
|
||||
|
||||
await sut.handleBackupDatabase();
|
||||
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalled();
|
||||
const call = mocks.process.spawnDuplexStream.mock.calls[0];
|
||||
const args = call[1] as string[];
|
||||
expect(args).toMatchInlineSnapshot(`
|
||||
[
|
||||
"postgresql://postgres:pwd@host:5432/immich?sslmode=require",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
it('should run a database backup successfully', async () => {
|
||||
const result = await sut.handleBackupDatabase();
|
||||
expect(result).toBe(JobStatus.Success);
|
||||
expect(mocks.storage.createWriteStream).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should rename file on success', async () => {
|
||||
const result = await sut.handleBackupDatabase();
|
||||
expect(result).toBe(JobStatus.Success);
|
||||
expect(mocks.storage.rename).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fail if pg_dump fails', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 1, '', 'error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
|
||||
});
|
||||
|
||||
it('should not rename file if pgdump fails and gzip succeeds', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 1, '', 'error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
|
||||
expect(mocks.storage.rename).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fail if gzip fails', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('gzip', 1, '', 'error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('gzip non-zero exit code (1)');
|
||||
});
|
||||
|
||||
it('should fail if write stream fails', async () => {
|
||||
mocks.storage.createWriteStream.mockImplementation(() => {
|
||||
throw new Error('error');
|
||||
});
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('error');
|
||||
});
|
||||
|
||||
it('should fail if rename fails', async () => {
|
||||
mocks.storage.rename.mockRejectedValue(new Error('error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('error');
|
||||
});
|
||||
|
||||
it('should ignore unlink failing and still return failed job status', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex('pg_dump', 1, '', 'error'));
|
||||
mocks.storage.unlink.mockRejectedValue(new Error('error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
|
||||
expect(mocks.storage.unlink).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each`
|
||||
postgresVersion | expectedVersion
|
||||
${'14.10'} | ${14}
|
||||
${'14.10.3'} | ${14}
|
||||
${'14.10 (Debian 14.10-1.pgdg120+1)'} | ${14}
|
||||
${'15.3.3'} | ${15}
|
||||
${'16.4.2'} | ${16}
|
||||
${'17.15.1'} | ${17}
|
||||
${'18.0.0'} | ${18}
|
||||
`(
|
||||
`should use pg_dump $expectedVersion with postgres version $postgresVersion`,
|
||||
async ({ postgresVersion, expectedVersion }) => {
|
||||
mocks.database.getPostgresVersion.mockResolvedValue(postgresVersion);
|
||||
await sut.handleBackupDatabase();
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledWith(
|
||||
`/usr/lib/postgresql/${expectedVersion}/bin/pg_dump`,
|
||||
expect.any(Array),
|
||||
expect.any(Object),
|
||||
);
|
||||
},
|
||||
);
|
||||
it.each`
|
||||
postgresVersion
|
||||
${'13.99.99'}
|
||||
${'19.0.0'}
|
||||
`(`should fail if postgres version $postgresVersion is not supported`, async ({ postgresVersion }) => {
|
||||
mocks.database.getPostgresVersion.mockResolvedValue(postgresVersion);
|
||||
const result = await sut.handleBackupDatabase();
|
||||
expect(mocks.process.spawn).not.toHaveBeenCalled();
|
||||
expect(result).toBe(JobStatus.Failed);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,99 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import path from 'node:path';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { OnEvent, OnJob } from 'src/decorators';
|
||||
import { DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum';
|
||||
import { ArgOf } from 'src/repositories/event.repository';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import {
|
||||
createDatabaseBackup,
|
||||
isFailedDatabaseBackupName,
|
||||
isValidDatabaseRoutineBackupName,
|
||||
UnsupportedPostgresError,
|
||||
} from 'src/utils/database-backups';
|
||||
import { handlePromiseError } from 'src/utils/misc';
|
||||
|
||||
@Injectable()
|
||||
export class BackupService extends BaseService {
|
||||
private backupLock = false;
|
||||
|
||||
@OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] })
|
||||
async onConfigInit({
|
||||
newConfig: {
|
||||
backup: { database },
|
||||
},
|
||||
}: ArgOf<'ConfigInit'>) {
|
||||
this.backupLock = await this.databaseRepository.tryLock(DatabaseLock.BackupDatabase);
|
||||
|
||||
if (this.backupLock) {
|
||||
this.cronRepository.create({
|
||||
name: 'backupDatabase',
|
||||
expression: database.cronExpression,
|
||||
onTick: () => handlePromiseError(this.jobRepository.queue({ name: JobName.DatabaseBackup }), this.logger),
|
||||
start: database.enabled,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'ConfigUpdate', server: true })
|
||||
onConfigUpdate({ newConfig: { backup } }: ArgOf<'ConfigUpdate'>) {
|
||||
if (!this.backupLock) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.cronRepository.update({
|
||||
name: 'backupDatabase',
|
||||
expression: backup.database.cronExpression,
|
||||
start: backup.database.enabled,
|
||||
});
|
||||
}
|
||||
|
||||
async cleanupDatabaseBackups() {
|
||||
this.logger.debug(`Database Backup Cleanup Started`);
|
||||
const {
|
||||
backup: { database: config },
|
||||
} = await this.getConfig({ withCache: false });
|
||||
|
||||
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
|
||||
const files = await this.storageRepository.readdir(backupsFolder);
|
||||
const backups = files
|
||||
.filter((filename) => isValidDatabaseRoutineBackupName(filename))
|
||||
.toSorted()
|
||||
.toReversed();
|
||||
const failedBackups = files.filter((filename) => isFailedDatabaseBackupName(filename));
|
||||
|
||||
const toDelete = backups.slice(config.keepLastAmount);
|
||||
toDelete.push(...failedBackups);
|
||||
|
||||
for (const file of toDelete) {
|
||||
await this.storageRepository.unlink(path.join(backupsFolder, file));
|
||||
}
|
||||
this.logger.debug(`Database Backup Cleanup Finished, deleted ${toDelete.length} backups`);
|
||||
}
|
||||
|
||||
@OnJob({ name: JobName.DatabaseBackup, queue: QueueName.BackupDatabase })
|
||||
async handleBackupDatabase(): Promise<JobStatus> {
|
||||
try {
|
||||
await createDatabaseBackup(this.backupRepos);
|
||||
} catch (error) {
|
||||
if (error instanceof UnsupportedPostgresError) {
|
||||
return JobStatus.Failed;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
await this.cleanupDatabaseBackups();
|
||||
return JobStatus.Success;
|
||||
}
|
||||
|
||||
private get backupRepos() {
|
||||
return {
|
||||
logger: this.logger,
|
||||
storage: this.storageRepository,
|
||||
config: this.configRepository,
|
||||
process: this.processRepository,
|
||||
database: this.databaseRepository,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,594 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { DateTime } from 'luxon';
|
||||
import { PassThrough, Readable } from 'node:stream';
|
||||
import { defaults, SystemConfig } from 'src/config';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { StorageFolder } from 'src/enum';
|
||||
import { ImmichWorker, JobStatus, StorageFolder } from 'src/enum';
|
||||
import { MaintenanceHealthRepository } from 'src/maintenance/maintenance-health.repository';
|
||||
import { DatabaseBackupService } from 'src/services/database-backup.service';
|
||||
import { MaintenanceService } from 'src/services/maintenance.service';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
import { systemConfigStub } from 'test/fixtures/system-config.stub';
|
||||
import { automock, AutoMocked, getMocks, mockDuplex, mockSpawn, ServiceMocks } from 'test/utils';
|
||||
|
||||
describe(MaintenanceService.name, () => {
|
||||
describe(DatabaseBackupService.name, () => {
|
||||
let sut: DatabaseBackupService;
|
||||
let mocks: ServiceMocks;
|
||||
let maintenanceHealthRepositoryMock: AutoMocked<MaintenanceHealthRepository>;
|
||||
|
||||
beforeEach(() => {
|
||||
({ sut, mocks } = newTestService(DatabaseBackupService));
|
||||
mocks = getMocks();
|
||||
maintenanceHealthRepositoryMock = automock(MaintenanceHealthRepository, {
|
||||
args: [mocks.logger],
|
||||
strict: false,
|
||||
});
|
||||
sut = new DatabaseBackupService(
|
||||
mocks.logger as never,
|
||||
mocks.storage as never,
|
||||
mocks.config,
|
||||
mocks.systemMetadata as never,
|
||||
mocks.process,
|
||||
mocks.database as never,
|
||||
mocks.cron as never,
|
||||
mocks.job as never,
|
||||
maintenanceHealthRepositoryMock as never,
|
||||
);
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('onBootstrapEvent', () => {
|
||||
it('should init cron job and handle config changes', async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(true);
|
||||
mocks.cron.create.mockResolvedValue();
|
||||
|
||||
await sut.onConfigInit({ newConfig: systemConfigStub.backupEnabled as SystemConfig });
|
||||
|
||||
expect(mocks.cron.create).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not initialize backup database cron job when lock is taken', async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(false);
|
||||
|
||||
await sut.onConfigInit({ newConfig: systemConfigStub.backupEnabled as SystemConfig });
|
||||
|
||||
expect(mocks.cron.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not initialise backup database job when running on microservices', async () => {
|
||||
mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices);
|
||||
await sut.onConfigInit({ newConfig: systemConfigStub.backupEnabled as SystemConfig });
|
||||
|
||||
expect(mocks.cron.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('onConfigUpdateEvent', () => {
|
||||
beforeEach(async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(true);
|
||||
mocks.cron.create.mockResolvedValue();
|
||||
|
||||
await sut.onConfigInit({ newConfig: defaults });
|
||||
});
|
||||
|
||||
it('should update cron job if backup is enabled', () => {
|
||||
mocks.cron.update.mockResolvedValue();
|
||||
|
||||
sut.onConfigUpdate({
|
||||
oldConfig: defaults,
|
||||
newConfig: {
|
||||
backup: {
|
||||
database: {
|
||||
enabled: true,
|
||||
cronExpression: '0 1 * * *',
|
||||
},
|
||||
},
|
||||
} as SystemConfig,
|
||||
});
|
||||
|
||||
expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'backupDatabase', expression: '0 1 * * *', start: true });
|
||||
expect(mocks.cron.update).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should do nothing if instance does not have the backup database lock', async () => {
|
||||
mocks.database.tryLock.mockResolvedValue(false);
|
||||
await sut.onConfigInit({ newConfig: defaults });
|
||||
sut.onConfigUpdate({ newConfig: systemConfigStub.backupEnabled as SystemConfig, oldConfig: defaults });
|
||||
expect(mocks.cron.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupDatabaseBackups', () => {
|
||||
it('should do nothing if not reached keepLastAmount', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.readdir.mockResolvedValue(['immich-db-backup-1.sql.gz']);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should remove failed backup files', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
//`immich-db-backup-${DateTime.now().toFormat("yyyyLLdd'T'HHmmss")}-v${serverVersion.toString()}-pg${databaseVersion.split(' ')[0]}.sql.gz.tmp`,
|
||||
mocks.storage.readdir.mockResolvedValue([
|
||||
'immich-db-backup-123.sql.gz.tmp',
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-25T11:02:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-27T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-29T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
]);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-123.sql.gz.tmp`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250725T110216-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250729T110116-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
);
|
||||
});
|
||||
|
||||
it('should remove old backup files over keepLastAmount', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.readdir.mockResolvedValue(['immich-db-backup-1.sql.gz', 'immich-db-backup-2.sql.gz']);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-1.sql.gz`,
|
||||
);
|
||||
});
|
||||
|
||||
it('should remove old backup files over keepLastAmount and failed backups', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.readdir.mockResolvedValue([
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-25T11:02:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-27T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
|
||||
'immich-db-backup-1753789649000.sql.gz',
|
||||
`immich-db-backup-${DateTime.fromISO('2025-07-29T11:01:16Z').toFormat("yyyyLLdd'T'HHmmss")}-v1.234.5-pg14.5.sql.gz`,
|
||||
]);
|
||||
await sut.cleanupDatabaseBackups();
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-1753789649000.sql.gz`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250725T110216-v1.234.5-pg14.5.sql.gz.tmp`,
|
||||
);
|
||||
expect(mocks.storage.unlink).toHaveBeenCalledWith(
|
||||
`${StorageCore.getBaseFolder(StorageFolder.Backups)}/immich-db-backup-20250727T110116-v1.234.5-pg14.5.sql.gz`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleBackupDatabase / createDatabaseBackup', () => {
|
||||
beforeEach(() => {
|
||||
mocks.storage.readdir.mockResolvedValue([]);
|
||||
mocks.process.spawn.mockReturnValue(mockSpawn(0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.storage.rename.mockResolvedValue();
|
||||
mocks.storage.unlink.mockResolvedValue();
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.createWriteStream.mockReturnValue(new PassThrough());
|
||||
});
|
||||
|
||||
it('should sanitize DB_URL (remove uselibpqcompat) before calling pg_dumpall', async () => {
|
||||
// create a service instance with a URL connection that includes libpqcompat
|
||||
const dbUrl = 'postgresql://postgres:pwd@host:5432/immich?sslmode=require&uselibpqcompat=true';
|
||||
const configMock = {
|
||||
getEnv: () => ({ database: { config: { connectionType: 'url', url: dbUrl }, skipMigrations: false } }),
|
||||
getWorker: () => ImmichWorker.Api,
|
||||
isDev: () => false,
|
||||
} as unknown as any;
|
||||
|
||||
sut = new DatabaseBackupService(
|
||||
mocks.logger as never,
|
||||
mocks.storage as never,
|
||||
configMock as never,
|
||||
mocks.systemMetadata as never,
|
||||
mocks.process,
|
||||
mocks.database as never,
|
||||
mocks.cron as never,
|
||||
mocks.job as never,
|
||||
void 0 as never,
|
||||
);
|
||||
|
||||
mocks.storage.readdir.mockResolvedValue([]);
|
||||
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.storage.rename.mockResolvedValue();
|
||||
mocks.storage.unlink.mockResolvedValue();
|
||||
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.backupEnabled);
|
||||
mocks.storage.createWriteStream.mockReturnValue(new PassThrough());
|
||||
mocks.database.getPostgresVersion.mockResolvedValue('14.10');
|
||||
|
||||
await sut.handleBackupDatabase();
|
||||
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalled();
|
||||
const call = mocks.process.spawnDuplexStream.mock.calls[0];
|
||||
const args = call[1] as string[];
|
||||
expect(args).toMatchInlineSnapshot(`
|
||||
[
|
||||
"postgresql://postgres:pwd@host:5432/immich?sslmode=require",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
it('should run a database backup successfully', async () => {
|
||||
const result = await sut.handleBackupDatabase();
|
||||
expect(result).toBe(JobStatus.Success);
|
||||
expect(mocks.storage.createWriteStream).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should rename file on success', async () => {
|
||||
const result = await sut.handleBackupDatabase();
|
||||
expect(result).toBe(JobStatus.Success);
|
||||
expect(mocks.storage.rename).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fail if pg_dump fails', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 1, '', 'error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
|
||||
});
|
||||
|
||||
it('should not rename file if pgdump fails and gzip succeeds', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 1, '', 'error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
|
||||
expect(mocks.storage.rename).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fail if gzip fails', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('gzip', 1, '', 'error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('gzip non-zero exit code (1)');
|
||||
});
|
||||
|
||||
it('should fail if write stream fails', async () => {
|
||||
mocks.storage.createWriteStream.mockImplementation(() => {
|
||||
throw new Error('error');
|
||||
});
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('error');
|
||||
});
|
||||
|
||||
it('should fail if rename fails', async () => {
|
||||
mocks.storage.rename.mockRejectedValue(new Error('error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('error');
|
||||
});
|
||||
|
||||
it('should ignore unlink failing and still return failed job status', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 1, '', 'error'));
|
||||
mocks.storage.unlink.mockRejectedValue(new Error('error'));
|
||||
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
|
||||
expect(mocks.storage.unlink).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each`
|
||||
postgresVersion | expectedVersion
|
||||
${'14.10'} | ${14}
|
||||
${'14.10.3'} | ${14}
|
||||
${'14.10 (Debian 14.10-1.pgdg120+1)'} | ${14}
|
||||
${'15.3.3'} | ${15}
|
||||
${'16.4.2'} | ${16}
|
||||
${'17.15.1'} | ${17}
|
||||
${'18.0.0'} | ${18}
|
||||
`(
|
||||
`should use pg_dump $expectedVersion with postgres version $postgresVersion`,
|
||||
async ({ postgresVersion, expectedVersion }) => {
|
||||
mocks.database.getPostgresVersion.mockResolvedValue(postgresVersion);
|
||||
await sut.handleBackupDatabase();
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledWith(
|
||||
`/usr/lib/postgresql/${expectedVersion}/bin/pg_dump`,
|
||||
expect.any(Array),
|
||||
expect.any(Object),
|
||||
);
|
||||
},
|
||||
);
|
||||
it.each`
|
||||
postgresVersion
|
||||
${'13.99.99'}
|
||||
${'19.0.0'}
|
||||
`(`should fail if postgres version $postgresVersion is not supported`, async ({ postgresVersion }) => {
|
||||
mocks.database.getPostgresVersion.mockResolvedValue(postgresVersion);
|
||||
const result = await sut.handleBackupDatabase();
|
||||
expect(mocks.process.spawn).not.toHaveBeenCalled();
|
||||
expect(result).toBe(JobStatus.Failed);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildPostgresLaunchArguments', () => {
|
||||
describe('default config', () => {
|
||||
it('should generate pg_dump arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('pg_dump')).resolves.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--username",
|
||||
"postgres",
|
||||
"--host",
|
||||
"database",
|
||||
"--port",
|
||||
"5432",
|
||||
"immich",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/pg_dump",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "postgres",
|
||||
"databaseUsername": "postgres",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it('should generate psql arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('psql')).resolves.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--username",
|
||||
"postgres",
|
||||
"--host",
|
||||
"database",
|
||||
"--port",
|
||||
"5432",
|
||||
"--dbname",
|
||||
"immich",
|
||||
"--echo-all",
|
||||
"--output=/dev/null",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/psql",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "postgres",
|
||||
"databaseUsername": "postgres",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it('should generate psql (single transaction) arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('psql', { singleTransaction: true })).resolves
|
||||
.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--username",
|
||||
"postgres",
|
||||
"--host",
|
||||
"database",
|
||||
"--port",
|
||||
"5432",
|
||||
"--dbname",
|
||||
"immich",
|
||||
"--single-transaction",
|
||||
"--set",
|
||||
"ON_ERROR_STOP=on",
|
||||
"--echo-all",
|
||||
"--output=/dev/null",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/psql",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "postgres",
|
||||
"databaseUsername": "postgres",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('using custom parts', () => {
|
||||
beforeEach(() => {
|
||||
const configMock = {
|
||||
getEnv: () => ({
|
||||
database: {
|
||||
config: {
|
||||
connectionType: 'parts',
|
||||
host: 'myhost',
|
||||
port: 1234,
|
||||
username: 'mypg',
|
||||
password: 'mypwd',
|
||||
database: 'myimmich',
|
||||
},
|
||||
skipMigrations: false,
|
||||
},
|
||||
}),
|
||||
getWorker: () => ImmichWorker.Api,
|
||||
isDev: () => false,
|
||||
} as unknown as any;
|
||||
|
||||
sut = new DatabaseBackupService(
|
||||
mocks.logger as never,
|
||||
mocks.storage as never,
|
||||
configMock as never,
|
||||
mocks.systemMetadata as never,
|
||||
mocks.process,
|
||||
mocks.database as never,
|
||||
mocks.cron as never,
|
||||
mocks.job as never,
|
||||
void 0 as never,
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate pg_dump arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('pg_dump')).resolves.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--username",
|
||||
"mypg",
|
||||
"--host",
|
||||
"myhost",
|
||||
"--port",
|
||||
"1234",
|
||||
"myimmich",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/pg_dump",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "mypwd",
|
||||
"databaseUsername": "mypg",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it('should generate psql (single transaction) arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('psql', { singleTransaction: true })).resolves
|
||||
.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--username",
|
||||
"mypg",
|
||||
"--host",
|
||||
"myhost",
|
||||
"--port",
|
||||
"1234",
|
||||
"--dbname",
|
||||
"myimmich",
|
||||
"--single-transaction",
|
||||
"--set",
|
||||
"ON_ERROR_STOP=on",
|
||||
"--echo-all",
|
||||
"--output=/dev/null",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/psql",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "mypwd",
|
||||
"databaseUsername": "mypg",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('using URL', () => {
|
||||
beforeEach(() => {
|
||||
const dbUrl = 'postgresql://mypg:mypwd@myhost:1234/myimmich?sslmode=require&uselibpqcompat=true';
|
||||
const configMock = {
|
||||
getEnv: () => ({ database: { config: { connectionType: 'url', url: dbUrl }, skipMigrations: false } }),
|
||||
getWorker: () => ImmichWorker.Api,
|
||||
isDev: () => false,
|
||||
} as unknown as any;
|
||||
|
||||
sut = new DatabaseBackupService(
|
||||
mocks.logger as never,
|
||||
mocks.storage as never,
|
||||
configMock as never,
|
||||
mocks.systemMetadata as never,
|
||||
mocks.process,
|
||||
mocks.database as never,
|
||||
mocks.cron as never,
|
||||
mocks.job as never,
|
||||
void 0 as never,
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate pg_dump arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('pg_dump')).resolves.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"postgresql://mypg:mypwd@myhost:1234/myimmich?sslmode=require",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/pg_dump",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "mypwd",
|
||||
"databaseUsername": "mypg",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it('should generate psql (single transaction) arguments', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('psql', { singleTransaction: true })).resolves
|
||||
.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--dbname",
|
||||
"postgresql://mypg:mypwd@myhost:1234/myimmich?sslmode=require",
|
||||
"--single-transaction",
|
||||
"--set",
|
||||
"ON_ERROR_STOP=on",
|
||||
"--echo-all",
|
||||
"--output=/dev/null",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/psql",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "mypwd",
|
||||
"databaseUsername": "mypg",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('using bad URL', () => {
|
||||
beforeEach(() => {
|
||||
const dbUrl = 'post://gresql://mypg:myp@wd@myhos:t:1234/myimmich?sslmode=require&uselibpqcompat=true';
|
||||
const configMock = {
|
||||
getEnv: () => ({ database: { config: { connectionType: 'url', url: dbUrl }, skipMigrations: false } }),
|
||||
getWorker: () => ImmichWorker.Api,
|
||||
isDev: () => false,
|
||||
} as unknown as any;
|
||||
|
||||
sut = new DatabaseBackupService(
|
||||
mocks.logger as never,
|
||||
mocks.storage as never,
|
||||
configMock as never,
|
||||
mocks.systemMetadata as never,
|
||||
mocks.process,
|
||||
mocks.database as never,
|
||||
mocks.cron as never,
|
||||
mocks.job as never,
|
||||
void 0 as never,
|
||||
);
|
||||
});
|
||||
|
||||
it('should fallback to reasonable defaults', async () => {
|
||||
await expect(sut.buildPostgresLaunchArguments('psql')).resolves.toMatchInlineSnapshot(`
|
||||
{
|
||||
"args": [
|
||||
"--dbname",
|
||||
"post://gresql//mypg:myp@wd@myhos:t:1234/myimmich?sslmode=require",
|
||||
"--echo-all",
|
||||
"--output=/dev/null",
|
||||
],
|
||||
"bin": "/usr/lib/postgresql/14/bin/psql",
|
||||
"databaseMajorVersion": 14,
|
||||
"databasePassword": "",
|
||||
"databaseUsername": "",
|
||||
"databaseVersion": "14.10 (Debian 14.10-1.pgdg120+1)",
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadBackup', () => {
|
||||
it('should reject invalid file names', async () => {
|
||||
await expect(sut.uploadBackup({ originalname: 'invalid backup' } as never)).rejects.toThrowError(
|
||||
new BadRequestException('Invalid backup name!'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should write file', async () => {
|
||||
await sut.uploadBackup({ originalname: 'path.sql.gz', buffer: 'buffer' } as never);
|
||||
expect(mocks.storage.createOrOverwriteFile).toBeCalledWith('/data/backups/uploaded-path.sql.gz', 'buffer');
|
||||
});
|
||||
});
|
||||
|
||||
describe('downloadBackup', () => {
|
||||
it('should reject invalid file names', () => {
|
||||
expect(() => sut.downloadBackup('invalid backup')).toThrowError(new BadRequestException('Invalid backup name!'));
|
||||
});
|
||||
|
||||
it('should get backup path', () => {
|
||||
expect(sut.downloadBackup('hello.sql.gz')).toEqual(
|
||||
expect.objectContaining({
|
||||
path: '/data/backups/hello.sql.gz',
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listBackups', () => {
|
||||
it('should give us all backups', async () => {
|
||||
mocks.storage.readdir.mockResolvedValue([
|
||||
@@ -54,30 +625,233 @@ describe(MaintenanceService.name, () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadBackup', () => {
|
||||
it('should reject invalid file names', async () => {
|
||||
await expect(sut.uploadBackup({ originalname: 'invalid backup' } as never)).rejects.toThrowError(
|
||||
new BadRequestException('Invalid backup name!'),
|
||||
describe('restoreDatabaseBackup', () => {
|
||||
beforeEach(() => {
|
||||
mocks.storage.readdir.mockResolvedValue([]);
|
||||
mocks.process.spawn.mockReturnValue(mockSpawn(0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementation(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.process.fork.mockImplementation(() => mockSpawn(0, 'Immich Server is listening', ''));
|
||||
mocks.storage.rename.mockResolvedValue();
|
||||
mocks.storage.unlink.mockResolvedValue();
|
||||
mocks.storage.createPlainReadStream.mockReturnValue(Readable.from(mockData()));
|
||||
mocks.storage.createWriteStream.mockReturnValue(new PassThrough());
|
||||
mocks.storage.createGzip.mockReturnValue(new PassThrough());
|
||||
mocks.storage.createGunzip.mockReturnValue(new PassThrough());
|
||||
|
||||
const configMock = {
|
||||
getEnv: () => ({
|
||||
database: {
|
||||
config: {
|
||||
connectionType: 'parts',
|
||||
host: 'myhost',
|
||||
port: 1234,
|
||||
username: 'mypg',
|
||||
password: 'mypwd',
|
||||
database: 'myimmich',
|
||||
},
|
||||
skipMigrations: false,
|
||||
},
|
||||
}),
|
||||
getWorker: () => ImmichWorker.Api,
|
||||
isDev: () => false,
|
||||
} as unknown as any;
|
||||
|
||||
sut = new DatabaseBackupService(
|
||||
mocks.logger as never,
|
||||
mocks.storage as never,
|
||||
configMock as never,
|
||||
mocks.systemMetadata as never,
|
||||
mocks.process,
|
||||
mocks.database as never,
|
||||
mocks.cron as never,
|
||||
mocks.job as never,
|
||||
maintenanceHealthRepositoryMock,
|
||||
);
|
||||
});
|
||||
|
||||
it('should write file', async () => {
|
||||
await sut.uploadBackup({ originalname: 'path.sql.gz', buffer: 'buffer' } as never);
|
||||
expect(mocks.storage.createOrOverwriteFile).toBeCalledWith('/data/backups/uploaded-path.sql.gz', 'buffer');
|
||||
});
|
||||
});
|
||||
|
||||
describe('downloadBackup', () => {
|
||||
it('should reject invalid file names', () => {
|
||||
expect(() => sut.downloadBackup('invalid backup')).toThrowError(new BadRequestException('Invalid backup name!'));
|
||||
it('should fail to restore invalid backup', async () => {
|
||||
await expect(sut.restoreDatabaseBackup('filename')).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`[Error: Invalid backup file format!]`,
|
||||
);
|
||||
});
|
||||
|
||||
it('should get backup path', () => {
|
||||
expect(sut.downloadBackup('hello.sql.gz')).toEqual(
|
||||
it('should successfully restore a backup', async () => {
|
||||
let writtenToPsql = '';
|
||||
|
||||
mocks.process.spawnDuplexStream.mockImplementationOnce(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementationOnce(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementationOnce(() => {
|
||||
return mockDuplex((chunk) => (writtenToPsql += chunk))('command', 0, 'data', '');
|
||||
});
|
||||
|
||||
const progress = vitest.fn();
|
||||
await sut.restoreDatabaseBackup('development-filename.sql', progress);
|
||||
|
||||
expect(progress).toHaveBeenCalledWith('backup', 0.05);
|
||||
expect(progress).toHaveBeenCalledWith('migrations', 0.9);
|
||||
|
||||
expect(maintenanceHealthRepositoryMock.checkApiHealth).toHaveBeenCalled();
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledTimes(3);
|
||||
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith(
|
||||
expect.stringMatching('/bin/psql'),
|
||||
[
|
||||
'--username',
|
||||
'mypg',
|
||||
'--host',
|
||||
'myhost',
|
||||
'--port',
|
||||
'1234',
|
||||
'--dbname',
|
||||
'myimmich',
|
||||
'--single-transaction',
|
||||
'--set',
|
||||
'ON_ERROR_STOP=on',
|
||||
'--echo-all',
|
||||
'--output=/dev/null',
|
||||
],
|
||||
expect.objectContaining({
|
||||
path: '/data/backups/hello.sql.gz',
|
||||
env: expect.objectContaining({
|
||||
PATH: expect.any(String),
|
||||
PGPASSWORD: 'mypwd',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(writtenToPsql).toMatchInlineSnapshot(`
|
||||
"
|
||||
-- drop all other database connections
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = current_database()
|
||||
AND pid <> pg_backend_pid();
|
||||
|
||||
-- re-create the default schema
|
||||
DROP SCHEMA public CASCADE;
|
||||
CREATE SCHEMA public;
|
||||
|
||||
-- restore access to schema
|
||||
GRANT ALL ON SCHEMA public TO "mypg";
|
||||
GRANT ALL ON SCHEMA public TO public;
|
||||
SELECT 1;"
|
||||
`);
|
||||
});
|
||||
|
||||
it('should generate pg_dumpall specific SQL instructions', async () => {
|
||||
let writtenToPsql = '';
|
||||
|
||||
mocks.process.spawnDuplexStream.mockImplementationOnce(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementationOnce(() => mockDuplex()('command', 0, 'data', ''));
|
||||
mocks.process.spawnDuplexStream.mockImplementationOnce(() => {
|
||||
return mockDuplex((chunk) => (writtenToPsql += chunk))('command', 0, 'data', '');
|
||||
});
|
||||
|
||||
const progress = vitest.fn();
|
||||
await sut.restoreDatabaseBackup('development-v2.4.0-.sql', progress);
|
||||
|
||||
expect(progress).toHaveBeenCalledWith('backup', 0.05);
|
||||
expect(progress).toHaveBeenCalledWith('migrations', 0.9);
|
||||
|
||||
expect(maintenanceHealthRepositoryMock.checkApiHealth).toHaveBeenCalled();
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledTimes(3);
|
||||
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith(
|
||||
expect.stringMatching('/bin/psql'),
|
||||
[
|
||||
'--username',
|
||||
'mypg',
|
||||
'--host',
|
||||
'myhost',
|
||||
'--port',
|
||||
'1234',
|
||||
'--dbname',
|
||||
'myimmich',
|
||||
'--echo-all',
|
||||
'--output=/dev/null',
|
||||
],
|
||||
expect.objectContaining({
|
||||
env: expect.objectContaining({
|
||||
PATH: expect.any(String),
|
||||
PGPASSWORD: 'mypwd',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(writtenToPsql).toMatchInlineSnapshot(String.raw`
|
||||
"
|
||||
-- drop all other database connections
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = current_database()
|
||||
AND pid <> pg_backend_pid();
|
||||
|
||||
\c postgres
|
||||
SELECT 1;"
|
||||
`);
|
||||
});
|
||||
|
||||
it('should fail if backup creation fails', async () => {
|
||||
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 1, '', 'error'));
|
||||
|
||||
const progress = vitest.fn();
|
||||
await expect(sut.restoreDatabaseBackup('development-filename.sql', progress)).rejects
|
||||
.toThrowErrorMatchingInlineSnapshot(`
|
||||
[Error: pg_dump non-zero exit code (1)
|
||||
error]
|
||||
`);
|
||||
|
||||
expect(progress).toHaveBeenCalledWith('backup', 0.05);
|
||||
});
|
||||
|
||||
it('should fail if restore itself fails', async () => {
|
||||
mocks.process.spawnDuplexStream
|
||||
.mockReturnValueOnce(mockDuplex()('pg_dump', 0, 'data', ''))
|
||||
.mockReturnValueOnce(mockDuplex()('gzip', 0, 'data', ''))
|
||||
.mockReturnValueOnce(mockDuplex()('psql', 1, '', 'error'));
|
||||
|
||||
const progress = vitest.fn();
|
||||
await expect(sut.restoreDatabaseBackup('development-filename.sql', progress)).rejects
|
||||
.toThrowErrorMatchingInlineSnapshot(`
|
||||
[Error: psql non-zero exit code (1)
|
||||
error]
|
||||
`);
|
||||
|
||||
expect(progress).toHaveBeenCalledWith('backup', 0.05);
|
||||
});
|
||||
|
||||
it('should rollback if database migrations fail', async () => {
|
||||
mocks.database.runMigrations.mockRejectedValue(new Error('Migrations Error'));
|
||||
|
||||
const progress = vitest.fn();
|
||||
await expect(
|
||||
sut.restoreDatabaseBackup('development-filename.sql', progress),
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`[Error: Migrations Error]`);
|
||||
|
||||
expect(progress).toHaveBeenCalledWith('backup', 0.05);
|
||||
expect(progress).toHaveBeenCalledWith('migrations', 0.9);
|
||||
|
||||
expect(maintenanceHealthRepositoryMock.checkApiHealth).toHaveBeenCalledTimes(0);
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('should rollback if API healthcheck fails', async () => {
|
||||
maintenanceHealthRepositoryMock.checkApiHealth.mockRejectedValue(new Error('Health Error'));
|
||||
|
||||
const progress = vitest.fn();
|
||||
await expect(
|
||||
sut.restoreDatabaseBackup('development-filename.sql', progress),
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`[Error: Health Error]`);
|
||||
|
||||
expect(progress).toHaveBeenCalledWith('backup', 0.05);
|
||||
expect(progress).toHaveBeenCalledWith('migrations', 0.9);
|
||||
expect(progress).toHaveBeenCalledWith('rollback', 0);
|
||||
|
||||
expect(maintenanceHealthRepositoryMock.checkApiHealth).toHaveBeenCalled();
|
||||
expect(mocks.process.spawnDuplexStream).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function* mockData() {
|
||||
yield 'SELECT 1;';
|
||||
}
|
||||
|
||||
@@ -1,43 +1,560 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { BadRequestException, Injectable, Optional } from '@nestjs/common';
|
||||
import { debounce } from 'lodash';
|
||||
import { DateTime } from 'luxon';
|
||||
import path, { basename } from 'node:path';
|
||||
import { PassThrough, Readable, Writable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import semver from 'semver';
|
||||
import { serverVersion } from 'src/constants';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { OnEvent, OnJob } from 'src/decorators';
|
||||
import { DatabaseBackupListResponseDto } from 'src/dtos/database-backup.dto';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { CacheControl, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum';
|
||||
import { MaintenanceHealthRepository } from 'src/maintenance/maintenance-health.repository';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { CronRepository } from 'src/repositories/cron.repository';
|
||||
import { DatabaseRepository } from 'src/repositories/database.repository';
|
||||
import { ArgOf } from 'src/repositories/event.repository';
|
||||
import { JobRepository } from 'src/repositories/job.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { ProcessRepository } from 'src/repositories/process.repository';
|
||||
import { StorageRepository } from 'src/repositories/storage.repository';
|
||||
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
|
||||
import { getConfig } from 'src/utils/config';
|
||||
import {
|
||||
deleteDatabaseBackup,
|
||||
downloadDatabaseBackup,
|
||||
listDatabaseBackups,
|
||||
uploadDatabaseBackup,
|
||||
findDatabaseBackupVersion,
|
||||
isFailedDatabaseBackupName,
|
||||
isValidDatabaseBackupName,
|
||||
isValidDatabaseRoutineBackupName,
|
||||
UnsupportedPostgresError,
|
||||
} from 'src/utils/database-backups';
|
||||
import { ImmichFileResponse } from 'src/utils/file';
|
||||
import { handlePromiseError } from 'src/utils/misc';
|
||||
|
||||
/**
|
||||
* This service is available outside of maintenance mode to manage maintenance mode
|
||||
*/
|
||||
@Injectable()
|
||||
export class DatabaseBackupService extends BaseService {
|
||||
async listBackups(): Promise<DatabaseBackupListResponseDto> {
|
||||
const backups = await listDatabaseBackups(this.backupRepos);
|
||||
return { backups };
|
||||
export class DatabaseBackupService {
|
||||
constructor(
|
||||
private readonly logger: LoggingRepository,
|
||||
private readonly storageRepository: StorageRepository,
|
||||
private readonly configRepository: ConfigRepository,
|
||||
private readonly systemMetadataRepository: SystemMetadataRepository,
|
||||
private readonly processRepository: ProcessRepository,
|
||||
private readonly databaseRepository: DatabaseRepository,
|
||||
@Optional()
|
||||
private readonly cronRepository: CronRepository,
|
||||
@Optional()
|
||||
private readonly jobRepository: JobRepository,
|
||||
@Optional()
|
||||
private readonly maintenanceHealthRepository: MaintenanceHealthRepository,
|
||||
) {
|
||||
this.logger.setContext(this.constructor.name);
|
||||
}
|
||||
|
||||
deleteBackup(files: string[]): Promise<void> {
|
||||
return deleteDatabaseBackup(this.backupRepos, files);
|
||||
private backupLock = false;
|
||||
|
||||
@OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] })
|
||||
async onConfigInit({
|
||||
newConfig: {
|
||||
backup: { database },
|
||||
},
|
||||
}: ArgOf<'ConfigInit'>) {
|
||||
if (!this.cronRepository || !this.jobRepository) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.backupLock = await this.databaseRepository.tryLock(DatabaseLock.BackupDatabase);
|
||||
|
||||
if (this.backupLock) {
|
||||
this.cronRepository.create({
|
||||
name: 'backupDatabase',
|
||||
expression: database.cronExpression,
|
||||
onTick: () => handlePromiseError(this.jobRepository.queue({ name: JobName.DatabaseBackup }), this.logger),
|
||||
start: database.enabled,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'ConfigUpdate', server: true })
|
||||
onConfigUpdate({ newConfig: { backup } }: ArgOf<'ConfigUpdate'>) {
|
||||
if (!this.cronRepository || !this.jobRepository || !this.backupLock) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.cronRepository.update({
|
||||
name: 'backupDatabase',
|
||||
expression: backup.database.cronExpression,
|
||||
start: backup.database.enabled,
|
||||
});
|
||||
}
|
||||
|
||||
@OnJob({ name: JobName.DatabaseBackup, queue: QueueName.BackupDatabase })
|
||||
async handleBackupDatabase(): Promise<JobStatus> {
|
||||
try {
|
||||
await this.createDatabaseBackup();
|
||||
} catch (error) {
|
||||
if (error instanceof UnsupportedPostgresError) {
|
||||
return JobStatus.Failed;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
await this.cleanupDatabaseBackups();
|
||||
return JobStatus.Success;
|
||||
}
|
||||
|
||||
async buildPostgresLaunchArguments(
|
||||
bin: 'pg_dump' | 'pg_dumpall' | 'psql',
|
||||
options: {
|
||||
singleTransaction?: boolean;
|
||||
} = {},
|
||||
): Promise<{
|
||||
bin: string;
|
||||
args: string[];
|
||||
databaseUsername: string;
|
||||
databasePassword: string;
|
||||
databaseVersion: string;
|
||||
databaseMajorVersion?: number;
|
||||
}> {
|
||||
const {
|
||||
database: { config: databaseConfig },
|
||||
} = this.configRepository.getEnv();
|
||||
const isUrlConnection = databaseConfig.connectionType === 'url';
|
||||
|
||||
const databaseVersion = await this.databaseRepository.getPostgresVersion();
|
||||
const databaseSemver = semver.coerce(databaseVersion);
|
||||
const databaseMajorVersion = databaseSemver?.major;
|
||||
|
||||
const args: string[] = [];
|
||||
let databaseUsername;
|
||||
|
||||
if (isUrlConnection) {
|
||||
if (bin !== 'pg_dump') {
|
||||
args.push('--dbname');
|
||||
}
|
||||
|
||||
let url = databaseConfig.url;
|
||||
if (URL.canParse(databaseConfig.url)) {
|
||||
const parsedUrl = new URL(databaseConfig.url);
|
||||
// remove known bad parameters
|
||||
parsedUrl.searchParams.delete('uselibpqcompat');
|
||||
|
||||
databaseUsername = parsedUrl.username;
|
||||
url = parsedUrl.toString();
|
||||
}
|
||||
|
||||
// assume typical values if we can't parse URL or not present
|
||||
databaseUsername ??= 'postgres';
|
||||
|
||||
args.push(url);
|
||||
} else {
|
||||
databaseUsername = databaseConfig.username;
|
||||
|
||||
args.push(
|
||||
'--username',
|
||||
databaseUsername,
|
||||
'--host',
|
||||
databaseConfig.host,
|
||||
'--port',
|
||||
databaseConfig.port.toString(),
|
||||
);
|
||||
|
||||
switch (bin) {
|
||||
case 'pg_dumpall': {
|
||||
args.push('--database');
|
||||
break;
|
||||
}
|
||||
case 'psql': {
|
||||
args.push('--dbname');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
args.push(databaseConfig.database);
|
||||
}
|
||||
|
||||
switch (bin) {
|
||||
case 'pg_dump':
|
||||
case 'pg_dumpall': {
|
||||
args.push('--clean', '--if-exists');
|
||||
break;
|
||||
}
|
||||
case 'psql': {
|
||||
if (options.singleTransaction) {
|
||||
args.push(
|
||||
// don't commit any transaction on failure
|
||||
'--single-transaction',
|
||||
// exit with non-zero code on error
|
||||
'--set',
|
||||
'ON_ERROR_STOP=on',
|
||||
);
|
||||
}
|
||||
|
||||
args.push(
|
||||
// used for progress monitoring
|
||||
'--echo-all',
|
||||
'--output=/dev/null',
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!databaseMajorVersion || !databaseSemver || !semver.satisfies(databaseSemver, '>=14.0.0 <19.0.0')) {
|
||||
this.logger.error(`Database Restore Failure: Unsupported PostgreSQL version: ${databaseVersion}`);
|
||||
throw new UnsupportedPostgresError(databaseVersion);
|
||||
}
|
||||
|
||||
return {
|
||||
bin: `/usr/lib/postgresql/${databaseMajorVersion}/bin/${bin}`,
|
||||
args,
|
||||
databaseUsername,
|
||||
databasePassword: isUrlConnection ? new URL(databaseConfig.url).password : databaseConfig.password,
|
||||
databaseVersion,
|
||||
databaseMajorVersion,
|
||||
};
|
||||
}
|
||||
|
||||
async createDatabaseBackup(filenamePrefix: string = ''): Promise<string> {
|
||||
this.logger.debug(`Database Backup Started`);
|
||||
|
||||
const { bin, args, databasePassword, databaseVersion, databaseMajorVersion } =
|
||||
await this.buildPostgresLaunchArguments('pg_dump');
|
||||
|
||||
this.logger.log(`Database Backup Starting. Database Version: ${databaseMajorVersion}`);
|
||||
|
||||
const filename = `${filenamePrefix}immich-db-backup-${DateTime.now().toFormat("yyyyLLdd'T'HHmmss")}-v${serverVersion.toString()}-pg${databaseVersion.split(' ')[0]}.sql.gz`;
|
||||
const backupFilePath = path.join(StorageCore.getBaseFolder(StorageFolder.Backups), filename);
|
||||
const temporaryFilePath = `${backupFilePath}.tmp`;
|
||||
|
||||
try {
|
||||
const pgdump = this.processRepository.spawnDuplexStream(bin, args, {
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
PGPASSWORD: databasePassword,
|
||||
},
|
||||
});
|
||||
|
||||
const gzip = this.processRepository.spawnDuplexStream('gzip', ['--rsyncable']);
|
||||
const fileStream = this.storageRepository.createWriteStream(temporaryFilePath);
|
||||
|
||||
await pipeline(pgdump, gzip, fileStream);
|
||||
await this.storageRepository.rename(temporaryFilePath, backupFilePath);
|
||||
} catch (error) {
|
||||
this.logger.error(`Database Backup Failure: ${error}`);
|
||||
await this.storageRepository
|
||||
.unlink(temporaryFilePath)
|
||||
.catch((error) => this.logger.error(`Failed to delete failed backup file: ${error}`));
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.logger.log(`Database Backup Success`);
|
||||
return backupFilePath;
|
||||
}
|
||||
|
||||
async uploadBackup(file: Express.Multer.File): Promise<void> {
|
||||
return uploadDatabaseBackup(this.backupRepos, file);
|
||||
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
|
||||
const fn = basename(file.originalname);
|
||||
if (!isValidDatabaseBackupName(fn)) {
|
||||
throw new BadRequestException('Invalid backup name!');
|
||||
}
|
||||
|
||||
const filePath = path.join(backupsFolder, `uploaded-${fn}`);
|
||||
await this.storageRepository.createOrOverwriteFile(filePath, file.buffer);
|
||||
}
|
||||
|
||||
downloadBackup(fileName: string): ImmichFileResponse {
|
||||
return downloadDatabaseBackup(fileName);
|
||||
}
|
||||
if (!isValidDatabaseBackupName(fileName)) {
|
||||
throw new BadRequestException('Invalid backup name!');
|
||||
}
|
||||
|
||||
const filePath = path.join(StorageCore.getBaseFolder(StorageFolder.Backups), fileName);
|
||||
|
||||
private get backupRepos() {
|
||||
return {
|
||||
logger: this.logger,
|
||||
storage: this.storageRepository,
|
||||
config: this.configRepository,
|
||||
process: this.processRepository,
|
||||
database: this.databaseRepository,
|
||||
path: filePath,
|
||||
fileName,
|
||||
cacheControl: CacheControl.PrivateWithoutCache,
|
||||
contentType: fileName.endsWith('.gz') ? 'application/gzip' : 'application/sql',
|
||||
};
|
||||
}
|
||||
|
||||
async listBackups(): Promise<DatabaseBackupListResponseDto> {
|
||||
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
|
||||
const files = await this.storageRepository.readdir(backupsFolder);
|
||||
|
||||
const validFiles = files
|
||||
.filter((fn) => isValidDatabaseBackupName(fn))
|
||||
.toSorted((a, b) => (a.startsWith('uploaded-') === b.startsWith('uploaded-') ? a.localeCompare(b) : 1))
|
||||
.toReversed();
|
||||
|
||||
const backups = await Promise.all(
|
||||
validFiles.map(async (filename) => {
|
||||
const stats = await this.storageRepository.stat(path.join(backupsFolder, filename));
|
||||
return { filename, filesize: stats.size };
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
backups,
|
||||
};
|
||||
}
|
||||
|
||||
async deleteBackup(files: string[]): Promise<void> {
|
||||
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
|
||||
|
||||
if (files.some((filename) => !isValidDatabaseBackupName(filename))) {
|
||||
throw new BadRequestException('Invalid backup name!');
|
||||
}
|
||||
|
||||
await Promise.all(files.map((filename) => this.storageRepository.unlink(path.join(backupsFolder, filename))));
|
||||
}
|
||||
|
||||
async cleanupDatabaseBackups() {
|
||||
this.logger.debug(`Database Backup Cleanup Started`);
|
||||
const {
|
||||
backup: { database: config },
|
||||
} = await getConfig(
|
||||
{
|
||||
configRepo: this.configRepository,
|
||||
metadataRepo: this.systemMetadataRepository,
|
||||
logger: this.logger,
|
||||
},
|
||||
{
|
||||
withCache: false,
|
||||
},
|
||||
);
|
||||
|
||||
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
|
||||
const files = await this.storageRepository.readdir(backupsFolder);
|
||||
const backups = files
|
||||
.filter((filename) => isValidDatabaseRoutineBackupName(filename))
|
||||
.toSorted()
|
||||
.toReversed();
|
||||
const failedBackups = files.filter((filename) => isFailedDatabaseBackupName(filename));
|
||||
|
||||
const toDelete = backups.slice(config.keepLastAmount);
|
||||
toDelete.push(...failedBackups);
|
||||
|
||||
for (const file of toDelete) {
|
||||
await this.storageRepository.unlink(path.join(backupsFolder, file));
|
||||
}
|
||||
|
||||
this.logger.debug(`Database Backup Cleanup Finished, deleted ${toDelete.length} backups`);
|
||||
}
|
||||
|
||||
async restoreDatabaseBackup(
|
||||
filename: string,
|
||||
progressCb?: (action: 'backup' | 'restore' | 'migrations' | 'rollback', progress: number) => void,
|
||||
): Promise<void> {
|
||||
this.logger.debug(`Database Restore Started`);
|
||||
|
||||
let complete = false;
|
||||
try {
|
||||
if (!isValidDatabaseBackupName(filename)) {
|
||||
throw new Error('Invalid backup file format!');
|
||||
}
|
||||
|
||||
const backupFilePath = path.join(StorageCore.getBaseFolder(StorageFolder.Backups), filename);
|
||||
await this.storageRepository.stat(backupFilePath); // => check file exists
|
||||
|
||||
let isPgClusterDump = false;
|
||||
const version = findDatabaseBackupVersion(filename);
|
||||
if (version && semver.satisfies(version, '<= 2.4')) {
|
||||
isPgClusterDump = true;
|
||||
}
|
||||
|
||||
const { bin, args, databaseUsername, databasePassword, databaseMajorVersion } =
|
||||
await this.buildPostgresLaunchArguments('psql', {
|
||||
singleTransaction: !isPgClusterDump,
|
||||
});
|
||||
|
||||
progressCb?.('backup', 0.05);
|
||||
|
||||
const restorePointFilePath = await this.createDatabaseBackup('restore-point-');
|
||||
|
||||
this.logger.log(`Database Restore Starting. Database Version: ${databaseMajorVersion}`);
|
||||
|
||||
let inputStream: Readable;
|
||||
if (backupFilePath.endsWith('.gz')) {
|
||||
const fileStream = this.storageRepository.createPlainReadStream(backupFilePath);
|
||||
const gunzip = this.storageRepository.createGunzip();
|
||||
fileStream.pipe(gunzip);
|
||||
inputStream = gunzip;
|
||||
} else {
|
||||
inputStream = this.storageRepository.createPlainReadStream(backupFilePath);
|
||||
}
|
||||
|
||||
const sqlStream = Readable.from(sql(inputStream, databaseUsername, isPgClusterDump));
|
||||
const psql = this.processRepository.spawnDuplexStream(bin, args, {
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
PGPASSWORD: databasePassword,
|
||||
},
|
||||
});
|
||||
|
||||
const [progressSource, progressSink] = createSqlProgressStreams((progress) => {
|
||||
if (complete) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(`Restore progress ~ ${(progress * 100).toFixed(2)}%`);
|
||||
progressCb?.('restore', progress);
|
||||
});
|
||||
|
||||
await pipeline(sqlStream, progressSource, psql, progressSink);
|
||||
|
||||
try {
|
||||
progressCb?.('migrations', 0.9);
|
||||
await this.databaseRepository.runMigrations();
|
||||
await this.maintenanceHealthRepository.checkApiHealth();
|
||||
} catch (error) {
|
||||
progressCb?.('rollback', 0);
|
||||
|
||||
const fileStream = this.storageRepository.createPlainReadStream(restorePointFilePath);
|
||||
const gunzip = this.storageRepository.createGunzip();
|
||||
fileStream.pipe(gunzip);
|
||||
inputStream = gunzip;
|
||||
|
||||
const sqlStream = Readable.from(sqlRollback(inputStream, databaseUsername));
|
||||
const psql = this.processRepository.spawnDuplexStream(bin, args, {
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
PGPASSWORD: databasePassword,
|
||||
},
|
||||
});
|
||||
|
||||
const [progressSource, progressSink] = createSqlProgressStreams((progress) => {
|
||||
if (complete) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(`Rollback progress ~ ${(progress * 100).toFixed(2)}%`);
|
||||
progressCb?.('rollback', progress);
|
||||
});
|
||||
|
||||
await pipeline(sqlStream, progressSource, psql, progressSink);
|
||||
|
||||
throw error;
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Database Restore Failure: ${error}`);
|
||||
throw error;
|
||||
} finally {
|
||||
complete = true;
|
||||
}
|
||||
|
||||
this.logger.log(`Database Restore Success`);
|
||||
}
|
||||
}
|
||||
|
||||
const SQL_DROP_CONNECTIONS = `
|
||||
-- drop all other database connections
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = current_database()
|
||||
AND pid <> pg_backend_pid();
|
||||
`;
|
||||
|
||||
const SQL_RESET_SCHEMA = (username: string) => `
|
||||
-- re-create the default schema
|
||||
DROP SCHEMA public CASCADE;
|
||||
CREATE SCHEMA public;
|
||||
|
||||
-- restore access to schema
|
||||
GRANT ALL ON SCHEMA public TO "${username}";
|
||||
GRANT ALL ON SCHEMA public TO public;
|
||||
`;
|
||||
|
||||
async function* sql(inputStream: Readable, databaseUsername: string, isPgClusterDump: boolean) {
|
||||
yield SQL_DROP_CONNECTIONS;
|
||||
yield isPgClusterDump
|
||||
? // it is likely the dump contains SQL to try to drop the currently active
|
||||
// database to ensure we have a fresh slate; if the `postgres` database exists
|
||||
// then prefer to switch before continuing otherwise this will just silently fail
|
||||
String.raw`
|
||||
\c postgres
|
||||
`
|
||||
: SQL_RESET_SCHEMA(databaseUsername);
|
||||
|
||||
for await (const chunk of inputStream) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
|
||||
async function* sqlRollback(inputStream: Readable, databaseUsername: string) {
|
||||
yield SQL_DROP_CONNECTIONS;
|
||||
yield SQL_RESET_SCHEMA(databaseUsername);
|
||||
|
||||
for await (const chunk of inputStream) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
|
||||
function createSqlProgressStreams(cb: (progress: number) => void) {
|
||||
const STDIN_START_MARKER = new TextEncoder().encode('FROM stdin');
|
||||
const STDIN_END_MARKER = new TextEncoder().encode(String.raw`\.`);
|
||||
|
||||
let readingStdin = false;
|
||||
let sequenceIdx = 0;
|
||||
|
||||
let linesSent = 0;
|
||||
let linesProcessed = 0;
|
||||
|
||||
const startedAt = +Date.now();
|
||||
const cbDebounced = debounce(
|
||||
() => {
|
||||
const progress = source.writableEnded
|
||||
? Math.min(1, linesProcessed / linesSent)
|
||||
: // progress simulation while we're in an indeterminate state
|
||||
Math.min(0.3, 0.1 + (Date.now() - startedAt) / 1e4);
|
||||
cb(progress);
|
||||
},
|
||||
100,
|
||||
{
|
||||
maxWait: 100,
|
||||
},
|
||||
);
|
||||
|
||||
let lastByte = -1;
|
||||
const source = new PassThrough({
|
||||
transform(chunk, _encoding, callback) {
|
||||
for (const byte of chunk) {
|
||||
if (!readingStdin && byte === 10 && lastByte !== 10) {
|
||||
linesSent += 1;
|
||||
}
|
||||
|
||||
lastByte = byte;
|
||||
|
||||
const sequence = readingStdin ? STDIN_END_MARKER : STDIN_START_MARKER;
|
||||
if (sequence[sequenceIdx] === byte) {
|
||||
sequenceIdx += 1;
|
||||
|
||||
if (sequence.length === sequenceIdx) {
|
||||
sequenceIdx = 0;
|
||||
readingStdin = !readingStdin;
|
||||
}
|
||||
} else {
|
||||
sequenceIdx = 0;
|
||||
}
|
||||
}
|
||||
|
||||
cbDebounced();
|
||||
this.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
const sink = new Writable({
|
||||
write(chunk, _encoding, callback) {
|
||||
for (const byte of chunk) {
|
||||
if (byte === 10) {
|
||||
linesProcessed++;
|
||||
}
|
||||
}
|
||||
|
||||
cbDebounced();
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
return [source, sink];
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import { AssetService } from 'src/services/asset.service';
|
||||
import { AuditService } from 'src/services/audit.service';
|
||||
import { AuthAdminService } from 'src/services/auth-admin.service';
|
||||
import { AuthService } from 'src/services/auth.service';
|
||||
import { BackupService } from 'src/services/backup.service';
|
||||
import { CliService } from 'src/services/cli.service';
|
||||
import { DatabaseBackupService } from 'src/services/database-backup.service';
|
||||
import { DatabaseService } from 'src/services/database.service';
|
||||
@@ -58,7 +57,6 @@ export const services = [
|
||||
AuditService,
|
||||
AuthService,
|
||||
AuthAdminService,
|
||||
BackupService,
|
||||
CliService,
|
||||
DatabaseBackupService,
|
||||
DatabaseService,
|
||||
|
||||
Reference in New Issue
Block a user