feat: schema-check (#25904)

This commit is contained in:
Jason Rasmussen
2026-02-12 17:59:00 -05:00
committed by GitHub
parent 7413356a2f
commit 8ef4e4d452
37 changed files with 449 additions and 213 deletions

View File

@@ -9,6 +9,7 @@ import {
import { DisableOAuthLogin, EnableOAuthLogin } from 'src/commands/oauth-login';
import { DisablePasswordLoginCommand, EnablePasswordLoginCommand } from 'src/commands/password-login';
import { PromptPasswordQuestions, ResetAdminPasswordCommand } from 'src/commands/reset-admin-password.command';
import { SchemaCheck } from 'src/commands/schema-check';
import { VersionCommand } from 'src/commands/version.command';
export const commandsAndQuestions = [
@@ -28,4 +29,5 @@ export const commandsAndQuestions = [
ChangeMediaLocationCommand,
PromptMediaLocationQuestions,
PromptConfirmMoveQuestions,
SchemaCheck,
];

View File

@@ -0,0 +1,60 @@
import { Command, CommandRunner } from 'nest-commander';
import { ErrorMessages } from 'src/constants';
import { CliService } from 'src/services/cli.service';
import { asHuman } from 'src/sql-tools/schema-diff';
@Command({
name: 'schema-check',
description: 'Verify database migrations and check for schema drift',
})
export class SchemaCheck extends CommandRunner {
constructor(private service: CliService) {
super();
}
async run(): Promise<void> {
try {
const { migrations, drift } = await this.service.schemaReport();
if (migrations.every((item) => item.status === 'applied')) {
console.log('Migrations are up to date');
} else {
console.log('Migration issues detected:');
for (const migration of migrations) {
switch (migration.status) {
case 'deleted': {
console.log(` - ${migration.name} was applied, but the file no longer exists on disk`);
break;
}
case 'missing': {
console.log(` - ${migration.name} exists, but has not been applied to the database`);
break;
}
}
}
}
if (drift.items.length === 0) {
console.log('\nNo schema drift detected');
} else {
console.log(`\n${ErrorMessages.SchemaDrift}`);
for (const item of drift.items) {
console.log(` - ${item.type}: ` + asHuman(item));
}
console.log(`
The below SQL is automatically generated and may be helpful for resolving drift. ** Use at your own risk! **
\`\`\`sql
${drift.asSql().join('\n')}
\`\`\`
`);
}
} catch (error) {
console.error(error);
console.error('Unable to debug migrations');
}
}
}

View File

@@ -4,6 +4,13 @@ import { dirname, join } from 'node:path';
import { SemVer } from 'semver';
import { ApiTag, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
export const ErrorMessages = {
InconsistentMediaLocation:
'Detected an inconsistent media location. For more information, see https://docs.immich.app/errors#inconsistent-media-location',
SchemaDrift: `Detected schema drift. For more information, see https://docs.immich.app/errors#schema-drift`,
TypeOrmUpgrade: 'Invalid upgrade path. For more information, see https://docs.immich.app/errors/#typeorm-upgrade',
};
export const POSTGRES_VERSION_RANGE = '>=14.0.0';
export const VECTORCHORD_VERSION_RANGE = '>=0.3 <2';
export const VECTORS_VERSION_RANGE = '>=0.2 <0.4';

View File

@@ -19,7 +19,9 @@ import { GenerateSql } from 'src/decorators';
import { DatabaseExtension, DatabaseLock, VectorIndex } from 'src/enum';
import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import 'src/schema'; // make sure all schema definitions are imported for schemaFromCode
import { DB } from 'src/schema';
import { schemaDiff, schemaFromCode, schemaFromDatabase } from 'src/sql-tools';
import { ExtensionVersion, VectorExtension, VectorUpdateResult } from 'src/types';
import { vectorIndexQuery } from 'src/utils/database';
import { isValidInteger } from 'src/validation';
@@ -281,6 +283,27 @@ export class DatabaseRepository {
return rows[0].db;
}
getMigrations() {
return this.db.selectFrom('kysely_migrations').select(['name', 'timestamp']).orderBy('name', 'asc').execute();
}
async getSchemaDrift() {
const source = schemaFromCode({ overrides: true, namingStrategy: 'default' });
const target = await schemaFromDatabase(this.db, {});
const drift = schemaDiff(source, target, {
tables: { ignoreExtra: true },
constraints: { ignoreExtra: false },
indexes: { ignoreExtra: true },
triggers: { ignoreExtra: true },
columns: { ignoreExtra: true },
functions: { ignoreExtra: false },
parameters: { ignoreExtra: true },
});
return drift;
}
async getDimensionSize(table: string, column = 'embedding'): Promise<number> {
const { rows } = await sql<{ dimsize: number }>`
SELECT atttypmod as dimsize

View File

@@ -106,7 +106,7 @@ export class MetadataRepository {
readTags(path: string): Promise<ImmichTags> {
const args = mimeTypes.isVideo(path) ? ['-ee'] : [];
return this.exiftool.read(path, args).catch((error) => {
return this.exiftool.read(path, { readArgs: args }).catch((error) => {
this.logger.warn(`Error reading exif data (${path}): ${error}\n${error?.stack}`);
return {};
}) as Promise<ImmichTags>;

View File

@@ -168,6 +168,8 @@ export interface Migrations {
}
export interface DB {
kysely_migrations: { timestamp: string; name: string };
activity: ActivityTable;
album: AlbumTable;

View File

@@ -1,4 +1,5 @@
import { Kysely, sql } from 'kysely';
import { ErrorMessages } from 'src/constants';
import { DatabaseExtension } from 'src/enum';
import { getVectorExtension } from 'src/repositories/database.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
@@ -16,9 +17,7 @@ export async function up(db: Kysely<any>): Promise<void> {
rows: [lastMigration],
} = await lastMigrationSql.execute(db);
if (lastMigration?.name !== 'AddMissingIndex1744910873956') {
throw new Error(
'Invalid upgrade path. For more information, see https://docs.immich.app/errors/#typeorm-upgrade',
);
throw new Error(ErrorMessages.TypeOrmUpgrade);
}
logger.log('Database has up to date TypeORM migrations, skipping initial Kysely migration');
return;

View File

@@ -1,15 +1,59 @@
import { Injectable } from '@nestjs/common';
import { isAbsolute } from 'node:path';
import { isAbsolute, join } from 'node:path';
import { SALT_ROUNDS } from 'src/constants';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';
import { MaintenanceAction, SystemMetadataKey } from 'src/enum';
import { BaseService } from 'src/services/base.service';
import { schemaDiff } from 'src/sql-tools';
import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance';
import { getExternalDomain } from 'src/utils/misc';
export type SchemaReport = {
migrations: MigrationStatus[];
drift: ReturnType<typeof schemaDiff>;
};
type MigrationStatus = {
name: string;
status: 'applied' | 'missing' | 'deleted';
};
@Injectable()
export class CliService extends BaseService {
async schemaReport(): Promise<SchemaReport> {
// eslint-disable-next-line unicorn/prefer-module
const allFiles = await this.storageRepository.readdir(join(__dirname, '../schema/migrations'));
const files = allFiles.filter((file) => file.endsWith('.js')).map((file) => file.slice(0, -3));
const rows = await this.databaseRepository.getMigrations();
const filesSet = new Set(files);
const rowsSet = new Set(rows.map((item) => item.name));
const combined = [...filesSet, ...rowsSet].toSorted();
const migrations: MigrationStatus[] = [];
for (const name of combined) {
if (filesSet.has(name) && rowsSet.has(name)) {
migrations.push({ name, status: 'applied' });
continue;
}
if (filesSet.has(name) && !rowsSet.has(name)) {
migrations.push({ name, status: 'missing' });
continue;
}
if (!filesSet.has(name) && rowsSet.has(name)) {
migrations.push({ name, status: 'deleted' });
continue;
}
}
const drift = await this.databaseRepository.getSchemaDrift();
return { migrations, drift };
}
async listUsers(): Promise<UserAdminResponseDto[]> {
const users = await this.userRepository.getList({ withDeleted: true });
return users.map((user) => mapUserAdmin(user));

View File

@@ -21,6 +21,11 @@ describe(DatabaseService.name, () => {
extensionRange = '0.2.x';
mocks.database.getVectorExtension.mockResolvedValue(DatabaseExtension.VectorChord);
mocks.database.getExtensionVersionRange.mockReturnValue(extensionRange);
mocks.database.getSchemaDrift.mockResolvedValue({
items: [],
asSql: () => [],
asHuman: () => [],
});
versionBelowRange = '0.1.0';
minVersionInRange = '0.2.0';

View File

@@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import semver from 'semver';
import { EXTENSION_NAMES, VECTOR_EXTENSIONS } from 'src/constants';
import { ErrorMessages, EXTENSION_NAMES, VECTOR_EXTENSIONS } from 'src/constants';
import { OnEvent } from 'src/decorators';
import { BootstrapEventPriority, DatabaseExtension, DatabaseLock, VectorIndex } from 'src/enum';
import { BaseService } from 'src/services/base.service';
@@ -124,6 +124,17 @@ export class DatabaseService extends BaseService {
const { database } = this.configRepository.getEnv();
if (!database.skipMigrations) {
await this.databaseRepository.runMigrations();
this.logger.log('Checking for schema drift');
const drift = await this.databaseRepository.getSchemaDrift();
if (drift.items.length === 0) {
this.logger.log('No schema drift detected');
} else {
this.logger.warn(`${ErrorMessages.SchemaDrift} or run \`immich-admin schema-check\``);
for (const warning of drift.asHuman()) {
this.logger.warn(` - ${warning}`);
}
}
}
await Promise.all([
this.databaseRepository.prewarm(VectorIndex.Clip),

View File

@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
import { ErrorMessages } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent, OnJob } from 'src/decorators';
import {
@@ -114,9 +115,7 @@ export class StorageService extends BaseService {
this.logger.log(`Media location changed (from=${previous}, to=${current})`);
if (!path.startsWith(previous)) {
throw new Error(
'Detected an inconsistent media location. For more information, see https://docs.immich.app/errors#inconsistent-media-location',
);
throw new Error(ErrorMessages.InconsistentMediaLocation);
}
this.logger.warn(

View File

@@ -15,7 +15,7 @@ const testColumn: DatabaseColumn = {
describe('compareColumns', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareColumns.onExtra(testColumn)).toEqual([
expect(compareColumns().onExtra(testColumn)).toEqual([
{
tableName: 'table1',
columnName: 'test',
@@ -28,7 +28,7 @@ describe('compareColumns', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareColumns.onMissing(testColumn)).toEqual([
expect(compareColumns().onMissing(testColumn)).toEqual([
{
type: 'ColumnAdd',
column: testColumn,
@@ -40,14 +40,14 @@ describe('compareColumns', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareColumns.onCompare(testColumn, testColumn)).toEqual([]);
expect(compareColumns().onCompare(testColumn, testColumn)).toEqual([]);
});
it('should detect a change in type', () => {
const source: DatabaseColumn = { ...testColumn };
const target: DatabaseColumn = { ...testColumn, type: 'text' };
const reason = 'column type is different (character varying vs text)';
expect(compareColumns.onCompare(source, target)).toEqual([
expect(compareColumns().onCompare(source, target)).toEqual([
{
columnName: 'test',
tableName: 'table1',
@@ -66,7 +66,7 @@ describe('compareColumns', () => {
const source: DatabaseColumn = { ...testColumn, nullable: true };
const target: DatabaseColumn = { ...testColumn, nullable: true, default: "''" };
const reason = `default is different (null vs '')`;
expect(compareColumns.onCompare(source, target)).toEqual([
expect(compareColumns().onCompare(source, target)).toEqual([
{
columnName: 'test',
tableName: 'table1',
@@ -83,7 +83,7 @@ describe('compareColumns', () => {
const source: DatabaseColumn = { ...testColumn, comment: 'new comment' };
const target: DatabaseColumn = { ...testColumn, comment: 'old comment' };
const reason = 'comment is different (new comment vs old comment)';
expect(compareColumns.onCompare(source, target)).toEqual([
expect(compareColumns().onCompare(source, target)).toEqual([
{
columnName: 'test',
tableName: 'table1',

View File

@@ -1,98 +1,99 @@
import { asRenameKey, getColumnType, isDefaultEqual } from 'src/sql-tools/helpers';
import { Comparer, DatabaseColumn, Reason, SchemaDiff } from 'src/sql-tools/types';
export const compareColumns = {
getRenameKey: (column) => {
return asRenameKey([
column.tableName,
column.type,
column.nullable,
column.default,
column.storage,
column.primary,
column.isArray,
column.length,
column.identity,
column.enumName,
column.numericPrecision,
column.numericScale,
]);
},
onRename: (source, target) => [
{
type: 'ColumnRename',
tableName: source.tableName,
oldName: target.name,
newName: source.name,
reason: Reason.Rename,
export const compareColumns = () =>
({
getRenameKey: (column) => {
return asRenameKey([
column.tableName,
column.type,
column.nullable,
column.default,
column.storage,
column.primary,
column.isArray,
column.length,
column.identity,
column.enumName,
column.numericPrecision,
column.numericScale,
]);
},
],
onMissing: (source) => [
{
type: 'ColumnAdd',
column: source,
reason: Reason.MissingInTarget,
onRename: (source, target) => [
{
type: 'ColumnRename',
tableName: source.tableName,
oldName: target.name,
newName: source.name,
reason: Reason.Rename,
},
],
onMissing: (source) => [
{
type: 'ColumnAdd',
column: source,
reason: Reason.MissingInTarget,
},
],
onExtra: (target) => [
{
type: 'ColumnDrop',
tableName: target.tableName,
columnName: target.name,
reason: Reason.MissingInSource,
},
],
onCompare: (source, target) => {
const sourceType = getColumnType(source);
const targetType = getColumnType(target);
const isTypeChanged = sourceType !== targetType;
if (isTypeChanged) {
// TODO: convert between types via UPDATE when possible
return dropAndRecreateColumn(source, target, `column type is different (${sourceType} vs ${targetType})`);
}
const items: SchemaDiff[] = [];
if (source.nullable !== target.nullable) {
items.push({
type: 'ColumnAlter',
tableName: source.tableName,
columnName: source.name,
changes: {
nullable: source.nullable,
},
reason: `nullable is different (${source.nullable} vs ${target.nullable})`,
});
}
if (!isDefaultEqual(source, target)) {
items.push({
type: 'ColumnAlter',
tableName: source.tableName,
columnName: source.name,
changes: {
default: String(source.default ?? 'NULL'),
},
reason: `default is different (${source.default ?? 'null'} vs ${target.default})`,
});
}
if (source.comment !== target.comment) {
items.push({
type: 'ColumnAlter',
tableName: source.tableName,
columnName: source.name,
changes: {
comment: String(source.comment),
},
reason: `comment is different (${source.comment} vs ${target.comment})`,
});
}
return items;
},
],
onExtra: (target) => [
{
type: 'ColumnDrop',
tableName: target.tableName,
columnName: target.name,
reason: Reason.MissingInSource,
},
],
onCompare: (source, target) => {
const sourceType = getColumnType(source);
const targetType = getColumnType(target);
const isTypeChanged = sourceType !== targetType;
if (isTypeChanged) {
// TODO: convert between types via UPDATE when possible
return dropAndRecreateColumn(source, target, `column type is different (${sourceType} vs ${targetType})`);
}
const items: SchemaDiff[] = [];
if (source.nullable !== target.nullable) {
items.push({
type: 'ColumnAlter',
tableName: source.tableName,
columnName: source.name,
changes: {
nullable: source.nullable,
},
reason: `nullable is different (${source.nullable} vs ${target.nullable})`,
});
}
if (!isDefaultEqual(source, target)) {
items.push({
type: 'ColumnAlter',
tableName: source.tableName,
columnName: source.name,
changes: {
default: String(source.default ?? 'NULL'),
},
reason: `default is different (${source.default ?? 'null'} vs ${target.default})`,
});
}
if (source.comment !== target.comment) {
items.push({
type: 'ColumnAlter',
tableName: source.tableName,
columnName: source.name,
changes: {
comment: String(source.comment),
},
reason: `comment is different (${source.comment} vs ${target.comment})`,
});
}
return items;
},
} satisfies Comparer<DatabaseColumn>;
}) satisfies Comparer<DatabaseColumn>;
const dropAndRecreateColumn = (source: DatabaseColumn, target: DatabaseColumn, reason: string): SchemaDiff[] => {
return [

View File

@@ -13,7 +13,7 @@ const testConstraint: DatabaseConstraint = {
describe('compareConstraints', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareConstraints.onExtra(testConstraint)).toEqual([
expect(compareConstraints().onExtra(testConstraint)).toEqual([
{
type: 'ConstraintDrop',
constraintName: 'test',
@@ -26,7 +26,7 @@ describe('compareConstraints', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareConstraints.onMissing(testConstraint)).toEqual([
expect(compareConstraints().onMissing(testConstraint)).toEqual([
{
type: 'ConstraintAdd',
constraint: testConstraint,
@@ -38,14 +38,14 @@ describe('compareConstraints', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareConstraints.onCompare(testConstraint, testConstraint)).toEqual([]);
expect(compareConstraints().onCompare(testConstraint, testConstraint)).toEqual([]);
});
it('should detect a change in type', () => {
const source: DatabaseConstraint = { ...testConstraint };
const target: DatabaseConstraint = { ...testConstraint, columnNames: ['column1', 'column2'] };
const reason = 'Primary key columns are different: (column1 vs column1,column2)';
expect(compareConstraints.onCompare(source, target)).toEqual([
expect(compareConstraints().onCompare(source, target)).toEqual([
{
constraintName: 'test',
tableName: 'table1',

View File

@@ -12,7 +12,7 @@ import {
SchemaDiff,
} from 'src/sql-tools/types';
export const compareConstraints: Comparer<DatabaseConstraint> = {
export const compareConstraints = (): Comparer<DatabaseConstraint> => ({
getRenameKey: (constraint) => {
switch (constraint.type) {
case ConstraintType.PRIMARY_KEY:
@@ -83,7 +83,7 @@ export const compareConstraints: Comparer<DatabaseConstraint> = {
}
}
},
};
});
const comparePrimaryKeyConstraint: CompareFunction<DatabasePrimaryKeyConstraint> = (source, target) => {
if (!haveEqualColumns(source.columnNames, target.columnNames)) {

View File

@@ -7,7 +7,7 @@ const testEnum: DatabaseEnum = { name: 'test', values: ['foo', 'bar'], synchroni
describe('compareEnums', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareEnums.onExtra(testEnum)).toEqual([
expect(compareEnums().onExtra(testEnum)).toEqual([
{
enumName: 'test',
type: 'EnumDrop',
@@ -19,7 +19,7 @@ describe('compareEnums', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareEnums.onMissing(testEnum)).toEqual([
expect(compareEnums().onMissing(testEnum)).toEqual([
{
type: 'EnumCreate',
enum: testEnum,
@@ -31,13 +31,13 @@ describe('compareEnums', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareEnums.onCompare(testEnum, testEnum)).toEqual([]);
expect(compareEnums().onCompare(testEnum, testEnum)).toEqual([]);
});
it('should drop and recreate when values list is different', () => {
const source = { name: 'test', values: ['foo', 'bar'], synchronize: true };
const target = { name: 'test', values: ['foo', 'bar', 'world'], synchronize: true };
expect(compareEnums.onCompare(source, target)).toEqual([
expect(compareEnums().onCompare(source, target)).toEqual([
{
enumName: 'test',
type: 'EnumDrop',

View File

@@ -1,6 +1,6 @@
import { Comparer, DatabaseEnum, Reason } from 'src/sql-tools/types';
export const compareEnums: Comparer<DatabaseEnum> = {
export const compareEnums = (): Comparer<DatabaseEnum> => ({
onMissing: (source) => [
{
type: 'EnumCreate',
@@ -35,4 +35,4 @@ export const compareEnums: Comparer<DatabaseEnum> = {
return [];
},
};
});

View File

@@ -7,7 +7,7 @@ const testExtension = { name: 'test', synchronize: true };
describe('compareExtensions', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareExtensions.onExtra(testExtension)).toEqual([
expect(compareExtensions().onExtra(testExtension)).toEqual([
{
extensionName: 'test',
type: 'ExtensionDrop',
@@ -19,7 +19,7 @@ describe('compareExtensions', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareExtensions.onMissing(testExtension)).toEqual([
expect(compareExtensions().onMissing(testExtension)).toEqual([
{
type: 'ExtensionCreate',
extension: testExtension,
@@ -31,7 +31,7 @@ describe('compareExtensions', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareExtensions.onCompare(testExtension, testExtension)).toEqual([]);
expect(compareExtensions().onCompare(testExtension, testExtension)).toEqual([]);
});
});
});

View File

@@ -1,6 +1,6 @@
import { Comparer, DatabaseExtension, Reason } from 'src/sql-tools/types';
export const compareExtensions: Comparer<DatabaseExtension> = {
export const compareExtensions = (): Comparer<DatabaseExtension> => ({
onMissing: (source) => [
{
type: 'ExtensionCreate',
@@ -19,4 +19,4 @@ export const compareExtensions: Comparer<DatabaseExtension> = {
// if the name matches they are the same
return [];
},
};
});

View File

@@ -11,7 +11,7 @@ const testFunction: DatabaseFunction = {
describe('compareFunctions', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareFunctions.onExtra(testFunction)).toEqual([
expect(compareFunctions().onExtra(testFunction)).toEqual([
{
functionName: 'test',
type: 'FunctionDrop',
@@ -23,7 +23,7 @@ describe('compareFunctions', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareFunctions.onMissing(testFunction)).toEqual([
expect(compareFunctions().onMissing(testFunction)).toEqual([
{
type: 'FunctionCreate',
function: testFunction,
@@ -35,13 +35,13 @@ describe('compareFunctions', () => {
describe('onCompare', () => {
it('should ignore functions with the same hash', () => {
expect(compareFunctions.onCompare(testFunction, testFunction)).toEqual([]);
expect(compareFunctions().onCompare(testFunction, testFunction)).toEqual([]);
});
it('should report differences if functions have different hashes', () => {
const source: DatabaseFunction = { ...testFunction, expression: 'SELECT 1' };
const target: DatabaseFunction = { ...testFunction, expression: 'SELECT 2' };
expect(compareFunctions.onCompare(source, target)).toEqual([
expect(compareFunctions().onCompare(source, target)).toEqual([
{
type: 'FunctionCreate',
reason: 'function expression has changed (SELECT 1 vs SELECT 2)',

View File

@@ -1,6 +1,6 @@
import { Comparer, DatabaseFunction, Reason } from 'src/sql-tools/types';
export const compareFunctions: Comparer<DatabaseFunction> = {
export const compareFunctions = (): Comparer<DatabaseFunction> => ({
onMissing: (source) => [
{
type: 'FunctionCreate',
@@ -29,4 +29,4 @@ export const compareFunctions: Comparer<DatabaseFunction> = {
return [];
},
};
});

View File

@@ -13,7 +13,7 @@ const testIndex: DatabaseIndex = {
describe('compareIndexes', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareIndexes.onExtra(testIndex)).toEqual([
expect(compareIndexes().onExtra(testIndex)).toEqual([
{
type: 'IndexDrop',
indexName: 'test',
@@ -25,7 +25,7 @@ describe('compareIndexes', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareIndexes.onMissing(testIndex)).toEqual([
expect(compareIndexes().onMissing(testIndex)).toEqual([
{
type: 'IndexCreate',
index: testIndex,
@@ -37,7 +37,7 @@ describe('compareIndexes', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareIndexes.onCompare(testIndex, testIndex)).toEqual([]);
expect(compareIndexes().onCompare(testIndex, testIndex)).toEqual([]);
});
it('should drop and recreate when column list is different', () => {
@@ -55,7 +55,7 @@ describe('compareIndexes', () => {
unique: true,
synchronize: true,
};
expect(compareIndexes.onCompare(source, target)).toEqual([
expect(compareIndexes().onCompare(source, target)).toEqual([
{
indexName: 'test',
type: 'IndexDrop',

View File

@@ -1,7 +1,7 @@
import { asRenameKey, haveEqualColumns } from 'src/sql-tools/helpers';
import { Comparer, DatabaseIndex, Reason } from 'src/sql-tools/types';
export const compareIndexes: Comparer<DatabaseIndex> = {
export const compareIndexes = (): Comparer<DatabaseIndex> => ({
getRenameKey: (index) => {
if (index.override) {
return index.override.value.sql.replace(index.name, 'INDEX_NAME');
@@ -59,4 +59,4 @@ export const compareIndexes: Comparer<DatabaseIndex> = {
return [];
},
};
});

View File

@@ -11,7 +11,7 @@ const testOverride: DatabaseOverride = {
describe('compareOverrides', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareOverrides.onExtra(testOverride)).toEqual([
expect(compareOverrides().onExtra(testOverride)).toEqual([
{
type: 'OverrideDrop',
overrideName: 'test',
@@ -23,7 +23,7 @@ describe('compareOverrides', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareOverrides.onMissing(testOverride)).toEqual([
expect(compareOverrides().onMissing(testOverride)).toEqual([
{
type: 'OverrideCreate',
override: testOverride,
@@ -35,7 +35,7 @@ describe('compareOverrides', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareOverrides.onCompare(testOverride, testOverride)).toEqual([]);
expect(compareOverrides().onCompare(testOverride, testOverride)).toEqual([]);
});
it('should drop and recreate when the value changes', () => {
@@ -57,7 +57,7 @@ describe('compareOverrides', () => {
},
synchronize: true,
};
expect(compareOverrides.onCompare(source, target)).toEqual([
expect(compareOverrides().onCompare(source, target)).toEqual([
{
override: source,
type: 'OverrideUpdate',

View File

@@ -1,6 +1,6 @@
import { Comparer, DatabaseOverride, Reason } from 'src/sql-tools/types';
export const compareOverrides: Comparer<DatabaseOverride> = {
export const compareOverrides = (): Comparer<DatabaseOverride> => ({
onMissing: (source) => [
{
type: 'OverrideCreate',
@@ -26,4 +26,4 @@ export const compareOverrides: Comparer<DatabaseOverride> = {
return [];
},
};
});

View File

@@ -13,7 +13,7 @@ const testParameter: DatabaseParameter = {
describe('compareParameters', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareParameters.onExtra(testParameter)).toEqual([
expect(compareParameters().onExtra(testParameter)).toEqual([
{
type: 'ParameterReset',
databaseName: 'immich',
@@ -26,7 +26,7 @@ describe('compareParameters', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareParameters.onMissing(testParameter)).toEqual([
expect(compareParameters().onMissing(testParameter)).toEqual([
{
type: 'ParameterSet',
parameter: testParameter,
@@ -38,7 +38,7 @@ describe('compareParameters', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareParameters.onCompare(testParameter, testParameter)).toEqual([]);
expect(compareParameters().onCompare(testParameter, testParameter)).toEqual([]);
});
});
});

View File

@@ -1,6 +1,6 @@
import { Comparer, DatabaseParameter, Reason } from 'src/sql-tools/types';
export const compareParameters: Comparer<DatabaseParameter> = {
export const compareParameters = (): Comparer<DatabaseParameter> => ({
onMissing: (source) => [
{
type: 'ParameterSet',
@@ -20,4 +20,4 @@ export const compareParameters: Comparer<DatabaseParameter> = {
// TODO
return [];
},
};
});

View File

@@ -14,7 +14,7 @@ const testTable: DatabaseTable = {
describe('compareParameters', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareTables.onExtra(testTable)).toEqual([
expect(compareTables({}).onExtra(testTable)).toEqual([
{
type: 'TableDrop',
tableName: 'test',
@@ -26,7 +26,7 @@ describe('compareParameters', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareTables.onMissing(testTable)).toEqual([
expect(compareTables({}).onMissing(testTable)).toEqual([
{
type: 'TableCreate',
table: testTable,
@@ -38,7 +38,7 @@ describe('compareParameters', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareTables.onCompare(testTable, testTable)).toEqual([]);
expect(compareTables({}).onCompare(testTable, testTable)).toEqual([]);
});
});
});

View File

@@ -3,9 +3,9 @@ import { compareConstraints } from 'src/sql-tools/comparers/constraint.comparer'
import { compareIndexes } from 'src/sql-tools/comparers/index.comparer';
import { compareTriggers } from 'src/sql-tools/comparers/trigger.comparer';
import { compare } from 'src/sql-tools/helpers';
import { Comparer, DatabaseTable, Reason, SchemaDiff } from 'src/sql-tools/types';
import { Comparer, DatabaseTable, Reason, SchemaDiffOptions } from 'src/sql-tools/types';
export const compareTables: Comparer<DatabaseTable> = {
export const compareTables = (options: SchemaDiffOptions): Comparer<DatabaseTable> => ({
onMissing: (source) => [
{
type: 'TableCreate',
@@ -20,14 +20,12 @@ export const compareTables: Comparer<DatabaseTable> = {
reason: Reason.MissingInSource,
},
],
onCompare: (source, target) => compareTable(source, target),
};
const compareTable = (source: DatabaseTable, target: DatabaseTable): SchemaDiff[] => {
return [
...compare(source.columns, target.columns, {}, compareColumns),
...compare(source.indexes, target.indexes, {}, compareIndexes),
...compare(source.constraints, target.constraints, {}, compareConstraints),
...compare(source.triggers, target.triggers, {}, compareTriggers),
];
};
onCompare: (source, target) => {
return [
...compare(source.columns, target.columns, options.columns, compareColumns()),
...compare(source.indexes, target.indexes, options.indexes, compareIndexes()),
...compare(source.constraints, target.constraints, options.constraints, compareConstraints()),
...compare(source.triggers, target.triggers, options.triggers, compareTriggers()),
];
},
});

View File

@@ -15,7 +15,7 @@ const testTrigger: DatabaseTrigger = {
describe('compareTriggers', () => {
describe('onExtra', () => {
it('should work', () => {
expect(compareTriggers.onExtra(testTrigger)).toEqual([
expect(compareTriggers().onExtra(testTrigger)).toEqual([
{
type: 'TriggerDrop',
tableName: 'table1',
@@ -28,7 +28,7 @@ describe('compareTriggers', () => {
describe('onMissing', () => {
it('should work', () => {
expect(compareTriggers.onMissing(testTrigger)).toEqual([
expect(compareTriggers().onMissing(testTrigger)).toEqual([
{
type: 'TriggerCreate',
trigger: testTrigger,
@@ -40,49 +40,49 @@ describe('compareTriggers', () => {
describe('onCompare', () => {
it('should work', () => {
expect(compareTriggers.onCompare(testTrigger, testTrigger)).toEqual([]);
expect(compareTriggers().onCompare(testTrigger, testTrigger)).toEqual([]);
});
it('should detect a change in function name', () => {
const source: DatabaseTrigger = { ...testTrigger, functionName: 'my_new_name' };
const target: DatabaseTrigger = { ...testTrigger, functionName: 'my_old_name' };
const reason = `function is different (my_new_name vs my_old_name)`;
expect(compareTriggers.onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
expect(compareTriggers().onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
});
it('should detect a change in actions', () => {
const source: DatabaseTrigger = { ...testTrigger, actions: ['delete'] };
const target: DatabaseTrigger = { ...testTrigger, actions: ['delete', 'insert'] };
const reason = `action is different (delete vs delete,insert)`;
expect(compareTriggers.onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
expect(compareTriggers().onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
});
it('should detect a change in timing', () => {
const source: DatabaseTrigger = { ...testTrigger, timing: 'before' };
const target: DatabaseTrigger = { ...testTrigger, timing: 'after' };
const reason = `timing method is different (before vs after)`;
expect(compareTriggers.onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
expect(compareTriggers().onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
});
it('should detect a change in scope', () => {
const source: DatabaseTrigger = { ...testTrigger, scope: 'row' };
const target: DatabaseTrigger = { ...testTrigger, scope: 'statement' };
const reason = `scope is different (row vs statement)`;
expect(compareTriggers.onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
expect(compareTriggers().onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
});
it('should detect a change in new table reference', () => {
const source: DatabaseTrigger = { ...testTrigger, referencingNewTableAs: 'new_table' };
const target: DatabaseTrigger = { ...testTrigger, referencingNewTableAs: undefined };
const reason = `new table reference is different (new_table vs undefined)`;
expect(compareTriggers.onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
expect(compareTriggers().onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
});
it('should detect a change in old table reference', () => {
const source: DatabaseTrigger = { ...testTrigger, referencingOldTableAs: 'old_table' };
const target: DatabaseTrigger = { ...testTrigger, referencingOldTableAs: undefined };
const reason = `old table reference is different (old_table vs undefined)`;
expect(compareTriggers.onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
expect(compareTriggers().onCompare(source, target)).toEqual([{ type: 'TriggerCreate', trigger: source, reason }]);
});
});
});

View File

@@ -1,6 +1,6 @@
import { Comparer, DatabaseTrigger, Reason } from 'src/sql-tools/types';
export const compareTriggers: Comparer<DatabaseTrigger> = {
export const compareTriggers = (): Comparer<DatabaseTrigger> => ({
onMissing: (source) => [
{
type: 'TriggerCreate',
@@ -38,4 +38,4 @@ export const compareTriggers: Comparer<DatabaseTrigger> = {
return [];
},
};
});

View File

@@ -20,12 +20,12 @@ import {
*/
export const schemaDiff = (source: DatabaseSchema, target: DatabaseSchema, options: SchemaDiffOptions = {}) => {
const items = [
...compare(source.parameters, target.parameters, options.parameters, compareParameters),
...compare(source.extensions, target.extensions, options.extensions, compareExtensions),
...compare(source.functions, target.functions, options.functions, compareFunctions),
...compare(source.enums, target.enums, options.enums, compareEnums),
...compare(source.tables, target.tables, options.tables, compareTables),
...compare(source.overrides, target.overrides, options.overrides, compareOverrides),
...compare(source.parameters, target.parameters, options.parameters, compareParameters()),
...compare(source.extensions, target.extensions, options.extensions, compareExtensions()),
...compare(source.functions, target.functions, options.functions, compareFunctions()),
...compare(source.enums, target.enums, options.enums, compareEnums()),
...compare(source.tables, target.tables, options.tables, compareTables(options)),
...compare(source.overrides, target.overrides, options.overrides, compareOverrides()),
];
type SchemaName = SchemaDiff['type'];
@@ -103,6 +103,7 @@ export const schemaDiff = (source: DatabaseSchema, target: DatabaseSchema, optio
return {
items: orderedItems,
asSql: (options?: SchemaDiffToSqlOptions) => schemaDiffToSql(orderedItems, options),
asHuman: () => schemaDiffToHuman(orderedItems),
};
};
@@ -113,7 +114,14 @@ export const schemaDiffToSql = (items: SchemaDiff[], options: SchemaDiffToSqlOpt
return items.flatMap((item) => asSql(item, options));
};
const asSql = (item: SchemaDiff, options: SchemaDiffToSqlOptions): string[] => {
/**
* Convert schema diff into human readable statements
*/
export const schemaDiffToHuman = (items: SchemaDiff[]): string[] => {
return items.flatMap((item) => asHuman(item));
};
export const asSql = (item: SchemaDiff, options: SchemaDiffToSqlOptions): string[] => {
const ctx = new BaseContext(options);
for (const transform of transformers) {
const result = transform(ctx, item);
@@ -127,6 +135,88 @@ const asSql = (item: SchemaDiff, options: SchemaDiffToSqlOptions): string[] => {
throw new Error(`Unhandled schema diff type: ${item.type}`);
};
export const asHuman = (item: SchemaDiff): string => {
switch (item.type) {
case 'ExtensionCreate': {
return `The extension "${item.extension.name}" is missing and needs to be created`;
}
case 'ExtensionDrop': {
return `The extension "${item.extensionName}" exists but is no longer needed`;
}
case 'FunctionCreate': {
return `The function "${item.function.name}" is missing and needs to be created`;
}
case 'FunctionDrop': {
return `The function "${item.functionName}" exists but should be removed`;
}
case 'TableCreate': {
return `The table "${item.table.name}" is missing and needs to be created`;
}
case 'TableDrop': {
return `The table "${item.tableName}" exists but should be removed`;
}
case 'ColumnAdd': {
return `The column "${item.column.tableName}"."${item.column.name}" is missing and needs to be created`;
}
case 'ColumnRename': {
return `The column "${item.tableName}"."${item.oldName}" was renamed to "${item.tableName}"."${item.newName}"`;
}
case 'ColumnAlter': {
return `The column "${item.tableName}"."${item.columnName}" has changes that need to be applied ${JSON.stringify(
item.changes,
)}`;
}
case 'ColumnDrop': {
return `The column "${item.tableName}"."${item.columnName}" exists but should be removed`;
}
case 'ConstraintAdd': {
return `The constraint "${item.constraint.tableName}"."${item.constraint.name}" (${item.constraint.type}) is missing and needs to be created`;
}
case 'ConstraintRename': {
return `The constraint "${item.tableName}"."${item.oldName}" was renamed to "${item.tableName}"."${item.newName}"`;
}
case 'ConstraintDrop': {
return `The constraint "${item.tableName}"."${item.constraintName}" exists but should be removed`;
}
case 'IndexCreate': {
return `The index "${item.index.tableName}"."${item.index.name}" is missing and needs to be created`;
}
case 'IndexRename': {
return `The index "${item.tableName}"."${item.oldName}" was renamed to "${item.tableName}"."${item.newName}"`;
}
case 'IndexDrop': {
return `The index "${item.indexName}" exists but is no longer needed`;
}
case 'TriggerCreate': {
return `The trigger "${item.trigger.tableName}"."${item.trigger.name}" is missing and needs to be created`;
}
case 'TriggerDrop': {
return `The trigger "${item.tableName}"."${item.triggerName}" exists but is no longer needed`;
}
case 'ParameterSet': {
return `The configuration parameter "${item.parameter.name}" has a different value and needs to be updated to "${item.parameter.value}"`;
}
case 'ParameterReset': {
return `The configuration parameter "${item.parameterName}" is set, but should be reset to the default value`;
}
case 'EnumCreate': {
return `The enum "${item.enum.name}" is missing and needs to be created`;
}
case 'EnumDrop': {
return `The enum "${item.enumName}" exists but is no longer needed`;
}
case 'OverrideCreate': {
return `The override "${item.override.name}" is missing and needs to be created`;
}
case 'OverrideUpdate': {
return `The override "${item.override.name}" needs to be updated`;
}
case 'OverrideDrop': {
return `The override "${item.overrideName}" exists but is no longer needed`;
}
}
};
const withComments = (comments: boolean | undefined, item: SchemaDiff): string => {
if (!comments) {
return '';

View File

@@ -5,14 +5,20 @@ import { ReaderContext } from 'src/sql-tools/contexts/reader-context';
import { readers } from 'src/sql-tools/readers';
import { DatabaseSchema, PostgresDB, SchemaFromDatabaseOptions } from 'src/sql-tools/types';
export type DatabaseLike = Sql | Kysely<any>;
const isKysely = (db: DatabaseLike): db is Kysely<any> => db instanceof Kysely;
/**
* Load schema from a database url
*/
export const schemaFromDatabase = async (
postgres: Sql,
database: DatabaseLike,
options: SchemaFromDatabaseOptions = {},
): Promise<DatabaseSchema> => {
const db = new Kysely<PostgresDB>({ dialect: new PostgresJSDialect({ postgres }) });
const db = isKysely(database)
? (database as Kysely<PostgresDB>)
: new Kysely<PostgresDB>({ dialect: new PostgresJSDialect({ postgres: database }) });
const ctx = new ReaderContext(options);
try {
@@ -22,6 +28,9 @@ export const schemaFromDatabase = async (
return ctx.build();
} finally {
await db.destroy();
// only close the connection it we created it
if (!isKysely(database)) {
await db.destroy();
}
}
};

View File

@@ -30,6 +30,10 @@ export type SchemaDiffToSqlOptions = BaseContextOptions & {
export type SchemaDiffOptions = BaseContextOptions & {
tables?: IgnoreOptions;
columns?: IgnoreOptions;
indexes?: IgnoreOptions;
triggers?: IgnoreOptions;
constraints?: IgnoreOptions;
functions?: IgnoreOptions;
enums?: IgnoreOptions;
extensions?: IgnoreOptions;

View File

@@ -1,29 +0,0 @@
import { DatabaseRepository } from 'src/repositories/database.repository';
import { RepositoryInterface } from 'src/types';
import { Mocked, vitest } from 'vitest';
export const newDatabaseRepositoryMock = (): Mocked<RepositoryInterface<DatabaseRepository>> => {
return {
shutdown: vitest.fn(),
getExtensionVersions: vitest.fn(),
getVectorExtension: vitest.fn(),
getExtensionVersionRange: vitest.fn(),
getPostgresVersion: vitest.fn().mockResolvedValue('14.10 (Debian 14.10-1.pgdg120+1)'),
getPostgresVersionRange: vitest.fn().mockReturnValue('>=14.0.0'),
createExtension: vitest.fn().mockResolvedValue(void 0),
dropExtension: vitest.fn(),
updateVectorExtension: vitest.fn(),
reindexVectorsIfNeeded: vitest.fn(),
getDimensionSize: vitest.fn(),
setDimensionSize: vitest.fn(),
deleteAllSearchEmbeddings: vitest.fn(),
prewarm: vitest.fn(),
runMigrations: vitest.fn(),
revertLastMigration: vitest.fn(),
withLock: vitest.fn().mockImplementation((_, function_: <R>() => Promise<R>) => function_()),
tryLock: vitest.fn(),
isBusy: vitest.fn(),
wait: vitest.fn(),
migrateFilePaths: vitest.fn(),
};
};

View File

@@ -75,7 +75,6 @@ import { IAccessRepositoryMock, newAccessRepositoryMock } from 'test/repositorie
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
import { newConfigRepositoryMock } from 'test/repositories/config.repository.mock';
import { newCryptoRepositoryMock } from 'test/repositories/crypto.repository.mock';
import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
import { newJobRepositoryMock } from 'test/repositories/job.repository.mock';
import { newMediaRepositoryMock } from 'test/repositories/media.repository.mock';
import { newMetadataRepositoryMock } from 'test/repositories/metadata.repository.mock';
@@ -279,6 +278,14 @@ export const getMocks = () => {
const loggerMock = { setContext: () => {} };
const configMock = { getEnv: () => ({}) };
// eslint-disable-next-line no-sparse-arrays
const databaseMock = automock(DatabaseRepository, { args: [, loggerMock], strict: false });
databaseMock.withLock.mockImplementation((_type, fn) => fn());
databaseMock.getPostgresVersion = vitest.fn().mockResolvedValue('14.10 (Debian 14.10-1.pgdg120+1)');
databaseMock.getPostgresVersionRange = vitest.fn().mockReturnValue('>=14.0.0');
databaseMock.createExtension = vitest.fn().mockResolvedValue(void 0);
const mocks: ServiceMocks = {
access: newAccessRepositoryMock(),
// eslint-disable-next-line no-sparse-arrays
@@ -295,7 +302,7 @@ export const getMocks = () => {
assetJob: automock(AssetJobRepository),
app: automock(AppRepository, { strict: false }),
config: newConfigRepositoryMock(),
database: newDatabaseRepositoryMock(),
database: databaseMock,
downloadRepository: automock(DownloadRepository, { strict: false }),
duplicateRepository: automock(DuplicateRepository),
email: automock(EmailRepository, { args: [loggerMock] }),