mirror of
https://github.com/immich-app/immich.git
synced 2026-03-04 09:57:33 +03:00
refactor: migrate person repository to kysely (#15242)
* refactor: migrate person repository to kysely * `asVector` begone * linting * fix metadata faces * update test --------- Co-authored-by: Alex <alex.tran1502@gmail.com> Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
Daniel Dietzler
@@ -1,13 +1,13 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
import { ExpressionBuilder, Insertable, Kysely, SelectExpression, sql } from 'kysely';
|
||||
import { jsonObjectFrom } from 'kysely/helpers/postgres';
|
||||
import _ from 'lodash';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { AssetFaces, DB, FaceSearch, Person } from 'src/db';
|
||||
import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
|
||||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { FaceSearchEntity } from 'src/entities/face-search.entity';
|
||||
import { PersonEntity } from 'src/entities/person.entity';
|
||||
import { PaginationMode, SourceType } from 'src/enum';
|
||||
import { SourceType } from 'src/enum';
|
||||
import {
|
||||
AssetFaceId,
|
||||
DeleteFacesOptions,
|
||||
@@ -17,332 +17,418 @@ import {
|
||||
PersonNameSearchOptions,
|
||||
PersonSearchOptions,
|
||||
PersonStatistics,
|
||||
SelectFaceOptions,
|
||||
UnassignFacesOptions,
|
||||
UpdateFacesData,
|
||||
} from 'src/interfaces/person.interface';
|
||||
import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';
|
||||
import { DataSource, FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm';
|
||||
import { mapUpsertColumns } from 'src/utils/database';
|
||||
import { Paginated, PaginationOptions } from 'src/utils/pagination';
|
||||
import { FindOptionsRelations } from 'typeorm';
|
||||
|
||||
const withPerson = (eb: ExpressionBuilder<DB, 'asset_faces'>) => {
|
||||
return jsonObjectFrom(
|
||||
eb.selectFrom('person').selectAll('person').whereRef('person.id', '=', 'asset_faces.personId'),
|
||||
).as('person');
|
||||
};
|
||||
|
||||
const withAsset = (eb: ExpressionBuilder<DB, 'asset_faces'>) => {
|
||||
return jsonObjectFrom(
|
||||
eb.selectFrom('assets').selectAll('assets').whereRef('assets.id', '=', 'asset_faces.assetId'),
|
||||
).as('asset');
|
||||
};
|
||||
|
||||
const withFaceSearch = (eb: ExpressionBuilder<DB, 'asset_faces'>) => {
|
||||
return jsonObjectFrom(
|
||||
eb.selectFrom('face_search').selectAll('face_search').whereRef('face_search.faceId', '=', 'asset_faces.id'),
|
||||
).as('faceSearch');
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class PersonRepository implements IPersonRepository {
|
||||
constructor(
|
||||
@InjectDataSource() private dataSource: DataSource,
|
||||
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
|
||||
@InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
|
||||
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
|
||||
@InjectRepository(FaceSearchEntity) private faceSearchRepository: Repository<FaceSearchEntity>,
|
||||
@InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
|
||||
) {}
|
||||
constructor(@InjectKysely() private db: Kysely<DB>) {}
|
||||
|
||||
@GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] })
|
||||
async reassignFaces({ oldPersonId, faceIds, newPersonId }: UpdateFacesData): Promise<number> {
|
||||
const result = await this.assetFaceRepository
|
||||
.createQueryBuilder()
|
||||
.update()
|
||||
const result = await this.db
|
||||
.updateTable('asset_faces')
|
||||
.set({ personId: newPersonId })
|
||||
.where(_.omitBy({ personId: oldPersonId, id: faceIds ? In(faceIds) : undefined }, _.isUndefined))
|
||||
.execute();
|
||||
.$if(!!oldPersonId, (qb) => qb.where('asset_faces.personId', '=', oldPersonId!))
|
||||
.$if(!!faceIds, (qb) => qb.where('asset_faces.id', 'in', faceIds!))
|
||||
.executeTakeFirst();
|
||||
|
||||
return result.affected ?? 0;
|
||||
return Number(result.numChangedRows) ?? 0;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ sourceType: SourceType.EXIF }] })
|
||||
async unassignFaces({ sourceType }: UnassignFacesOptions): Promise<void> {
|
||||
await this.assetFaceRepository
|
||||
.createQueryBuilder()
|
||||
.update()
|
||||
await this.db
|
||||
.updateTable('asset_faces')
|
||||
.set({ personId: null })
|
||||
.where({ sourceType })
|
||||
.where('asset_faces.sourceType', '=', sourceType)
|
||||
.execute();
|
||||
|
||||
await this.vacuum({ reindexVectors: false });
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[{ id: DummyValue.UUID }]] })
|
||||
async delete(entities: PersonEntity[]): Promise<void> {
|
||||
await this.personRepository.remove(entities);
|
||||
if (entities.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.db
|
||||
.deleteFrom('person')
|
||||
.where(
|
||||
'person.id',
|
||||
'in',
|
||||
entities.map(({ id }) => id),
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ sourceType: SourceType.EXIF }] })
|
||||
async deleteFaces({ sourceType }: DeleteFacesOptions): Promise<void> {
|
||||
await this.assetFaceRepository
|
||||
.createQueryBuilder('asset_faces')
|
||||
.delete()
|
||||
.andWhere('sourceType = :sourceType', { sourceType })
|
||||
.execute();
|
||||
await this.db.deleteFrom('asset_faces').where('asset_faces.sourceType', '=', sourceType).execute();
|
||||
|
||||
await this.vacuum({ reindexVectors: sourceType === SourceType.MACHINE_LEARNING });
|
||||
}
|
||||
|
||||
getAllFaces(
|
||||
pagination: PaginationOptions,
|
||||
options: FindManyOptions<AssetFaceEntity> = {},
|
||||
): Paginated<AssetFaceEntity> {
|
||||
return paginate(this.assetFaceRepository, pagination, options);
|
||||
getAllFaces(options: Partial<AssetFaceEntity> = {}): AsyncIterableIterator<AssetFaceEntity> {
|
||||
return this.db
|
||||
.selectFrom('asset_faces')
|
||||
.selectAll('asset_faces')
|
||||
.$if(options.personId === null, (qb) => qb.where('asset_faces.personId', 'is', null))
|
||||
.$if(!!options.personId, (qb) => qb.where('asset_faces.personId', '=', options.personId!))
|
||||
.$if(!!options.sourceType, (qb) => qb.where('asset_faces.sourceType', '=', options.sourceType!))
|
||||
.$if(!!options.assetId, (qb) => qb.where('asset_faces.assetId', '=', options.assetId!))
|
||||
.$if(!!options.assetId, (qb) => qb.where('asset_faces.assetId', '=', options.assetId!))
|
||||
.stream() as AsyncIterableIterator<AssetFaceEntity>;
|
||||
}
|
||||
|
||||
getAll(pagination: PaginationOptions, options: FindManyOptions<PersonEntity> = {}): Paginated<PersonEntity> {
|
||||
return paginate(this.personRepository, pagination, options);
|
||||
getAll(options: Partial<PersonEntity> = {}): AsyncIterableIterator<PersonEntity> {
|
||||
return this.db
|
||||
.selectFrom('person')
|
||||
.selectAll('person')
|
||||
.$if(!!options.ownerId, (qb) => qb.where('person.ownerId', '=', options.ownerId!))
|
||||
.$if(!!options.thumbnailPath, (qb) => qb.where('person.thumbnailPath', '=', options.thumbnailPath!))
|
||||
.$if(options.faceAssetId === null, (qb) => qb.where('person.faceAssetId', 'is', null))
|
||||
.$if(!!options.faceAssetId, (qb) => qb.where('person.faceAssetId', '=', options.faceAssetId!))
|
||||
.$if(options.isHidden !== undefined, (qb) => qb.where('person.isHidden', '=', options.isHidden!))
|
||||
.stream() as AsyncIterableIterator<PersonEntity>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ take: 10, skip: 10 }, DummyValue.UUID] })
|
||||
async getAllForUser(
|
||||
pagination: PaginationOptions,
|
||||
userId: string,
|
||||
options?: PersonSearchOptions,
|
||||
): Paginated<PersonEntity> {
|
||||
const queryBuilder = this.personRepository
|
||||
.createQueryBuilder('person')
|
||||
.innerJoin('person.faces', 'face')
|
||||
.where('person.ownerId = :userId', { userId })
|
||||
.innerJoin('face.asset', 'asset')
|
||||
.andWhere('asset.isArchived = false')
|
||||
.orderBy('person.isHidden', 'ASC')
|
||||
.addOrderBy("NULLIF(person.name, '') IS NULL", 'ASC')
|
||||
.addOrderBy('COUNT(face.assetId)', 'DESC')
|
||||
.addOrderBy("NULLIF(person.name, '')", 'ASC', 'NULLS LAST')
|
||||
.addOrderBy('person.createdAt')
|
||||
.having("person.name != '' OR COUNT(face.assetId) >= :faces", { faces: options?.minimumFaceCount || 1 })
|
||||
.groupBy('person.id');
|
||||
if (options?.closestFaceAssetId) {
|
||||
const innerQueryBuilder = this.faceSearchRepository
|
||||
.createQueryBuilder('face_search')
|
||||
.select('embedding', 'embedding')
|
||||
.where('"face_search"."faceId" = "person"."faceAssetId"');
|
||||
const faceSelectQueryBuilder = this.faceSearchRepository
|
||||
.createQueryBuilder('face_search')
|
||||
.select('embedding', 'embedding')
|
||||
.where('"face_search"."faceId" = :faceId', { faceId: options.closestFaceAssetId });
|
||||
queryBuilder
|
||||
.orderBy('(' + innerQueryBuilder.getQuery() + ') <=> (' + faceSelectQueryBuilder.getQuery() + ')')
|
||||
.setParameters(faceSelectQueryBuilder.getParameters());
|
||||
const items = (await this.db
|
||||
.selectFrom('person')
|
||||
.selectAll('person')
|
||||
.innerJoin('asset_faces', 'asset_faces.personId', 'person.id')
|
||||
.innerJoin('assets', (join) =>
|
||||
join
|
||||
.onRef('asset_faces.assetId', '=', 'assets.id')
|
||||
.on('assets.isArchived', '=', false)
|
||||
.on('assets.deletedAt', 'is', null),
|
||||
)
|
||||
.where('person.ownerId', '=', userId)
|
||||
.orderBy('person.isHidden', 'asc')
|
||||
.orderBy(sql`NULLIF(person.name, '') is null`, 'asc')
|
||||
.orderBy((eb) => eb.fn.count('asset_faces.assetId'), 'desc')
|
||||
.orderBy(sql`NULLIF(person.name, '')`, sql`asc nulls last`)
|
||||
.orderBy('person.createdAt')
|
||||
.having((eb) =>
|
||||
eb.or([
|
||||
eb('person.name', '!=', ''),
|
||||
eb((innerEb) => innerEb.fn.count('asset_faces.assetId'), '>=', options?.minimumFaceCount || 1),
|
||||
]),
|
||||
)
|
||||
.groupBy('person.id')
|
||||
.$if(!!options?.closestFaceAssetId, (qb) =>
|
||||
qb.orderBy((eb) =>
|
||||
eb(
|
||||
(eb) =>
|
||||
eb
|
||||
.selectFrom('face_search')
|
||||
.select('face_search.embedding')
|
||||
.whereRef('face_search.faceId', '=', 'person.faceAssetId'),
|
||||
'<=>',
|
||||
(eb) =>
|
||||
eb
|
||||
.selectFrom('face_search')
|
||||
.select('face_search.embedding')
|
||||
.where('face_search.faceId', '=', options!.closestFaceAssetId!),
|
||||
),
|
||||
),
|
||||
)
|
||||
.$if(!options?.withHidden, (qb) => qb.where('person.isHidden', '=', false))
|
||||
.offset(pagination.skip ?? 0)
|
||||
.limit(pagination.take + 1)
|
||||
.execute()) as PersonEntity[];
|
||||
|
||||
if (items.length > pagination.take) {
|
||||
return { items: items.slice(0, -1), hasNextPage: true };
|
||||
}
|
||||
if (!options?.withHidden) {
|
||||
queryBuilder.andWhere('person.isHidden = false');
|
||||
}
|
||||
return paginatedBuilder(queryBuilder, {
|
||||
mode: PaginationMode.LIMIT_OFFSET,
|
||||
...pagination,
|
||||
});
|
||||
|
||||
return { items, hasNextPage: false };
|
||||
}
|
||||
|
||||
@GenerateSql()
|
||||
getAllWithoutFaces(): Promise<PersonEntity[]> {
|
||||
return this.personRepository
|
||||
.createQueryBuilder('person')
|
||||
.leftJoin('person.faces', 'face')
|
||||
.having('COUNT(face.assetId) = 0')
|
||||
return this.db
|
||||
.selectFrom('person')
|
||||
.selectAll('person')
|
||||
.leftJoin('asset_faces', 'asset_faces.personId', 'person.id')
|
||||
.having((eb) => eb.fn.count('asset_faces.assetId'), '=', 0)
|
||||
.groupBy('person.id')
|
||||
.withDeleted()
|
||||
.getMany();
|
||||
.execute() as Promise<PersonEntity[]>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
getFaces(assetId: string): Promise<AssetFaceEntity[]> {
|
||||
return this.assetFaceRepository.find({
|
||||
where: { assetId },
|
||||
relations: {
|
||||
person: true,
|
||||
},
|
||||
order: {
|
||||
boundingBoxX1: 'ASC',
|
||||
},
|
||||
});
|
||||
return this.db
|
||||
.selectFrom('asset_faces')
|
||||
.selectAll('asset_faces')
|
||||
.select(withPerson)
|
||||
.where('asset_faces.assetId', '=', assetId)
|
||||
.orderBy('asset_faces.boundingBoxX1', 'asc')
|
||||
.execute() as Promise<AssetFaceEntity[]>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
getFaceById(id: string): Promise<AssetFaceEntity> {
|
||||
// TODO return null instead of find or fail
|
||||
return this.assetFaceRepository.findOneOrFail({
|
||||
where: { id },
|
||||
relations: {
|
||||
person: true,
|
||||
},
|
||||
});
|
||||
return this.db
|
||||
.selectFrom('asset_faces')
|
||||
.selectAll('asset_faces')
|
||||
.select(withPerson)
|
||||
.where('asset_faces.id', '=', id)
|
||||
.executeTakeFirstOrThrow() as Promise<AssetFaceEntity>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
getFaceByIdWithAssets(
|
||||
id: string,
|
||||
relations: FindOptionsRelations<AssetFaceEntity>,
|
||||
select: FindOptionsSelect<AssetFaceEntity>,
|
||||
relations?: FindOptionsRelations<AssetFaceEntity>,
|
||||
select?: SelectFaceOptions,
|
||||
): Promise<AssetFaceEntity | null> {
|
||||
return this.assetFaceRepository.findOne(
|
||||
_.omitBy(
|
||||
{
|
||||
where: { id },
|
||||
relations: {
|
||||
...relations,
|
||||
person: true,
|
||||
asset: true,
|
||||
},
|
||||
select,
|
||||
},
|
||||
_.isUndefined,
|
||||
),
|
||||
);
|
||||
return (this.db
|
||||
.selectFrom('asset_faces')
|
||||
.$if(!!select, (qb) =>
|
||||
qb.select(
|
||||
Object.keys(
|
||||
_.omitBy({ ...select!, faceSearch: undefined, asset: undefined }, _.isUndefined),
|
||||
) as SelectExpression<DB, 'asset_faces'>[],
|
||||
),
|
||||
)
|
||||
.$if(!select, (qb) => qb.selectAll('asset_faces'))
|
||||
.select(withPerson)
|
||||
.select(withAsset)
|
||||
.$if(!!relations?.faceSearch, (qb) => qb.select(withFaceSearch))
|
||||
.where('asset_faces.id', '=', id)
|
||||
.executeTakeFirst() ?? null) as Promise<AssetFaceEntity | null>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
|
||||
async reassignFace(assetFaceId: string, newPersonId: string): Promise<number> {
|
||||
const result = await this.assetFaceRepository
|
||||
.createQueryBuilder()
|
||||
.update()
|
||||
const result = await this.db
|
||||
.updateTable('asset_faces')
|
||||
.set({ personId: newPersonId })
|
||||
.where({ id: assetFaceId })
|
||||
.execute();
|
||||
.where('asset_faces.id', '=', assetFaceId)
|
||||
.executeTakeFirst();
|
||||
|
||||
return result.affected ?? 0;
|
||||
return Number(result.numChangedRows) ?? 0;
|
||||
}
|
||||
|
||||
getById(personId: string): Promise<PersonEntity | null> {
|
||||
return this.personRepository.findOne({ where: { id: personId } });
|
||||
return (this.db //
|
||||
.selectFrom('person')
|
||||
.selectAll('person')
|
||||
.where('person.id', '=', personId)
|
||||
.executeTakeFirst() ?? null) as Promise<PersonEntity | null>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING, { withHidden: true }] })
|
||||
getByName(userId: string, personName: string, { withHidden }: PersonNameSearchOptions): Promise<PersonEntity[]> {
|
||||
const queryBuilder = this.personRepository
|
||||
.createQueryBuilder('person')
|
||||
.where(
|
||||
'person.ownerId = :userId AND (LOWER(person.name) LIKE :nameStart OR LOWER(person.name) LIKE :nameAnywhere)',
|
||||
{ userId, nameStart: `${personName.toLowerCase()}%`, nameAnywhere: `% ${personName.toLowerCase()}%` },
|
||||
return this.db
|
||||
.selectFrom('person')
|
||||
.selectAll('person')
|
||||
.where((eb) =>
|
||||
eb.and([
|
||||
eb('person.ownerId', '=', userId),
|
||||
eb.or([
|
||||
eb(eb.fn('lower', ['person.name']), 'like', `${personName.toLowerCase()}%`),
|
||||
eb(eb.fn('lower', ['person.name']), 'like', `% ${personName.toLowerCase()}%`),
|
||||
]),
|
||||
]),
|
||||
)
|
||||
.limit(1000);
|
||||
|
||||
if (!withHidden) {
|
||||
queryBuilder.andWhere('person.isHidden = false');
|
||||
}
|
||||
return queryBuilder.getMany();
|
||||
.limit(1000)
|
||||
.$if(!withHidden, (qb) => qb.where('person.isHidden', '=', false))
|
||||
.execute() as Promise<PersonEntity[]>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, { withHidden: true }] })
|
||||
getDistinctNames(userId: string, { withHidden }: PersonNameSearchOptions): Promise<PersonNameResponse[]> {
|
||||
const queryBuilder = this.personRepository
|
||||
.createQueryBuilder('person')
|
||||
return this.db
|
||||
.selectFrom('person')
|
||||
.select(['person.id', 'person.name'])
|
||||
.distinctOn(['lower(person.name)'])
|
||||
.where(`person.ownerId = :userId AND person.name != ''`, { userId });
|
||||
|
||||
if (!withHidden) {
|
||||
queryBuilder.andWhere('person.isHidden = false');
|
||||
}
|
||||
|
||||
return queryBuilder.getMany();
|
||||
.distinctOn((eb) => eb.fn('lower', ['person.name']))
|
||||
.where((eb) => eb.and([eb('person.ownerId', '=', userId), eb('person.name', '!=', '')]))
|
||||
.$if(!withHidden, (qb) => qb.where('person.isHidden', '=', false))
|
||||
.execute();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
async getStatistics(personId: string): Promise<PersonStatistics> {
|
||||
const items = await this.assetFaceRepository
|
||||
.createQueryBuilder('face')
|
||||
.leftJoin('face.asset', 'asset')
|
||||
.where('face.personId = :personId', { personId })
|
||||
.andWhere('asset.isArchived = false')
|
||||
.andWhere('asset.deletedAt IS NULL')
|
||||
.andWhere('asset.livePhotoVideoId IS NULL')
|
||||
.select('COUNT(DISTINCT(asset.id))', 'count')
|
||||
.getRawOne();
|
||||
const result = await this.db
|
||||
.selectFrom('asset_faces')
|
||||
.leftJoin('assets', (join) =>
|
||||
join
|
||||
.onRef('assets.id', '=', 'asset_faces.assetId')
|
||||
.on('asset_faces.personId', '=', personId)
|
||||
.on('assets.isArchived', '=', false)
|
||||
.on('assets.deletedAt', 'is', null)
|
||||
.on('assets.livePhotoVideoId', 'is', null),
|
||||
)
|
||||
.select((eb) => eb.fn.count(eb.fn('distinct', ['assets.id'])).as('count'))
|
||||
.executeTakeFirst();
|
||||
|
||||
return {
|
||||
assets: items.count ?? 0,
|
||||
assets: result ? Number(result.count) : 0,
|
||||
};
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
async getNumberOfPeople(userId: string): Promise<PeopleStatistics> {
|
||||
const items = await this.personRepository
|
||||
.createQueryBuilder('person')
|
||||
.innerJoin('person.faces', 'face')
|
||||
.where('person.ownerId = :userId', { userId })
|
||||
.innerJoin('face.asset', 'asset')
|
||||
.andWhere('asset.isArchived = false')
|
||||
.select('COUNT(DISTINCT(person.id))', 'total')
|
||||
.addSelect('COUNT(DISTINCT(person.id)) FILTER (WHERE person.isHidden = true)', 'hidden')
|
||||
.getRawOne();
|
||||
const items = await this.db
|
||||
.selectFrom('person')
|
||||
.innerJoin('asset_faces', 'asset_faces.personId', 'person.id')
|
||||
.where('person.ownerId', '=', userId)
|
||||
.innerJoin('assets', (join) =>
|
||||
join
|
||||
.onRef('assets.id', '=', 'asset_faces.assetId')
|
||||
.on('assets.deletedAt', 'is', null)
|
||||
.on('assets.isArchived', '=', false),
|
||||
)
|
||||
.select((eb) => eb.fn.count(eb.fn('distinct', ['person.id'])).as('total'))
|
||||
.select((eb) =>
|
||||
eb.fn
|
||||
.count(eb.fn('distinct', ['person.id']))
|
||||
.filterWhere('person.isHidden', '=', true)
|
||||
.as('hidden'),
|
||||
)
|
||||
.executeTakeFirst();
|
||||
|
||||
if (items == undefined) {
|
||||
return { total: 0, hidden: 0 };
|
||||
}
|
||||
|
||||
const result: PeopleStatistics = {
|
||||
total: items.total ?? 0,
|
||||
hidden: items.hidden ?? 0,
|
||||
return {
|
||||
total: Number(items.total),
|
||||
hidden: Number(items.hidden),
|
||||
};
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
create(person: Partial<PersonEntity>): Promise<PersonEntity> {
|
||||
return this.save(person);
|
||||
create(person: Insertable<Person>): Promise<PersonEntity> {
|
||||
return this.db.insertInto('person').values(person).returningAll().executeTakeFirst() as Promise<PersonEntity>;
|
||||
}
|
||||
|
||||
async createAll(people: Partial<PersonEntity>[]): Promise<string[]> {
|
||||
const results = await this.personRepository.save(people);
|
||||
return results.map((person) => person.id);
|
||||
async createAll(people: Insertable<Person>[]): Promise<string[]> {
|
||||
const results = await this.db.insertInto('person').values(people).returningAll().execute();
|
||||
return results.map(({ id }) => id);
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[], [], [{ faceId: DummyValue.UUID, embedding: DummyValue.VECTOR }]] })
|
||||
async refreshFaces(
|
||||
facesToAdd: Partial<AssetFaceEntity>[],
|
||||
facesToAdd: (Insertable<AssetFaces> & { assetId: string })[],
|
||||
faceIdsToRemove: string[],
|
||||
embeddingsToAdd?: FaceSearchEntity[],
|
||||
embeddingsToAdd?: Insertable<FaceSearch>[],
|
||||
): Promise<void> {
|
||||
const query = this.faceSearchRepository.createQueryBuilder().select('1').fromDummy();
|
||||
let query = this.db;
|
||||
if (facesToAdd.length > 0) {
|
||||
const insertCte = this.assetFaceRepository.createQueryBuilder().insert().values(facesToAdd);
|
||||
query.addCommonTableExpression(insertCte, 'added');
|
||||
(query as any) = query.with('added', (db) => db.insertInto('asset_faces').values(facesToAdd));
|
||||
}
|
||||
|
||||
if (faceIdsToRemove.length > 0) {
|
||||
const deleteCte = this.assetFaceRepository
|
||||
.createQueryBuilder()
|
||||
.delete()
|
||||
.where('id = any(:faceIdsToRemove)', { faceIdsToRemove });
|
||||
query.addCommonTableExpression(deleteCte, 'deleted');
|
||||
(query as any) = query.with('removed', (db) =>
|
||||
db.deleteFrom('asset_faces').where('asset_faces.id', '=', (eb) => eb.fn.any(eb.val(faceIdsToRemove))),
|
||||
);
|
||||
}
|
||||
|
||||
if (embeddingsToAdd?.length) {
|
||||
const embeddingCte = this.faceSearchRepository.createQueryBuilder().insert().values(embeddingsToAdd).orIgnore();
|
||||
query.addCommonTableExpression(embeddingCte, 'embeddings');
|
||||
query.getQuery(); // typeorm mixes up parameters without this
|
||||
(query as any) = query.with('added_embeddings', (db) => db.insertInto('face_search').values(embeddingsToAdd));
|
||||
}
|
||||
|
||||
await query.execute();
|
||||
await query.selectFrom(sql`(select 1)`.as('dummy')).execute();
|
||||
}
|
||||
|
||||
async update(person: Partial<PersonEntity>): Promise<PersonEntity> {
|
||||
return this.save(person);
|
||||
async update(person: Partial<PersonEntity> & { id: string }): Promise<PersonEntity> {
|
||||
return this.db
|
||||
.updateTable('person')
|
||||
.set(person)
|
||||
.where('person.id', '=', person.id)
|
||||
.returningAll()
|
||||
.executeTakeFirstOrThrow() as Promise<PersonEntity>;
|
||||
}
|
||||
|
||||
async updateAll(people: Partial<PersonEntity>[]): Promise<void> {
|
||||
await this.personRepository.save(people);
|
||||
async updateAll(people: Insertable<Person>[]): Promise<void> {
|
||||
if (people.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.db
|
||||
.insertInto('person')
|
||||
.values(people)
|
||||
.onConflict((oc) => oc.column('id').doUpdateSet(() => mapUpsertColumns('person', people[0], ['id'])))
|
||||
.execute();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] })
|
||||
@ChunkedArray()
|
||||
async getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]> {
|
||||
return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true });
|
||||
getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]> {
|
||||
const { assetIds, personIds }: { assetIds: string[]; personIds: string[] } = { assetIds: [], personIds: [] };
|
||||
|
||||
for (const { assetId, personId } of ids) {
|
||||
assetIds.push(assetId);
|
||||
personIds.push(personId);
|
||||
}
|
||||
|
||||
return this.db
|
||||
.selectFrom('asset_faces')
|
||||
.selectAll('asset_faces')
|
||||
.select(withAsset)
|
||||
.select(withPerson)
|
||||
.where('asset_faces.assetId', 'in', assetIds)
|
||||
.where('asset_faces.personId', 'in', personIds)
|
||||
.execute() as Promise<AssetFaceEntity[]>;
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
async getRandomFace(personId: string): Promise<AssetFaceEntity | null> {
|
||||
return this.assetFaceRepository.findOneBy({ personId });
|
||||
getRandomFace(personId: string): Promise<AssetFaceEntity | null> {
|
||||
return (this.db
|
||||
.selectFrom('asset_faces')
|
||||
.selectAll('asset_faces')
|
||||
.where('asset_faces.personId', '=', personId)
|
||||
.executeTakeFirst() ?? null) as Promise<AssetFaceEntity | null>;
|
||||
}
|
||||
|
||||
@GenerateSql()
|
||||
async getLatestFaceDate(): Promise<string | undefined> {
|
||||
const result: { latestDate?: string } | undefined = await this.jobStatusRepository
|
||||
.createQueryBuilder('jobStatus')
|
||||
.select('MAX(jobStatus.facesRecognizedAt)::text', 'latestDate')
|
||||
.getRawOne();
|
||||
const result = (await this.db
|
||||
.selectFrom('asset_job_status')
|
||||
.select((eb) => sql`${eb.fn.max('asset_job_status.facesRecognizedAt')}::text`.as('latestDate'))
|
||||
.executeTakeFirst()) as { latestDate: string } | undefined;
|
||||
|
||||
return result?.latestDate;
|
||||
}
|
||||
|
||||
private async save(person: Partial<PersonEntity>): Promise<PersonEntity> {
|
||||
const { id } = await this.personRepository.save(person);
|
||||
return this.personRepository.findOneByOrFail({ id });
|
||||
}
|
||||
|
||||
private async vacuum({ reindexVectors }: { reindexVectors: boolean }): Promise<void> {
|
||||
await this.assetFaceRepository.query('VACUUM ANALYZE asset_faces, face_search, person');
|
||||
await this.assetFaceRepository.query('REINDEX TABLE asset_faces');
|
||||
await this.assetFaceRepository.query('REINDEX TABLE person');
|
||||
await sql`VACUUM ANALYZE asset_faces, face_search, person`.execute(this.db);
|
||||
await sql`REINDEX TABLE asset_faces`.execute(this.db);
|
||||
await sql`REINDEX TABLE person`.execute(this.db);
|
||||
if (reindexVectors) {
|
||||
await this.assetFaceRepository.query('REINDEX TABLE face_search');
|
||||
await sql`REINDEX TABLE face_search`.execute(this.db);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
SearchPaginationOptions,
|
||||
SmartSearchOptions,
|
||||
} from 'src/interfaces/search.interface';
|
||||
import { anyUuid, asUuid, asVector } from 'src/utils/database';
|
||||
import { anyUuid, asUuid } from 'src/utils/database';
|
||||
import { Paginated } from 'src/utils/pagination';
|
||||
import { isValidInteger } from 'src/validation';
|
||||
|
||||
@@ -82,7 +82,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
{ page: 1, size: 200 },
|
||||
{
|
||||
takenAfter: DummyValue.DATE,
|
||||
embedding: Array.from({ length: 512 }, Math.random),
|
||||
embedding: DummyValue.VECTOR,
|
||||
lensModel: DummyValue.STRING,
|
||||
withStacked: true,
|
||||
isFavorite: true,
|
||||
@@ -97,7 +97,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
|
||||
const items = (await searchAssetBuilder(this.db, options)
|
||||
.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
|
||||
.orderBy(sql`smart_search.embedding <=> ${asVector(options.embedding)}`)
|
||||
.orderBy(sql`smart_search.embedding <=> ${options.embedding}`)
|
||||
.limit(pagination.size + 1)
|
||||
.offset((pagination.page - 1) * pagination.size)
|
||||
.execute()) as any as AssetEntity[];
|
||||
@@ -111,7 +111,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
params: [
|
||||
{
|
||||
assetId: DummyValue.UUID,
|
||||
embedding: Array.from({ length: 512 }, Math.random),
|
||||
embedding: DummyValue.VECTOR,
|
||||
maxDistance: 0.6,
|
||||
type: AssetType.IMAGE,
|
||||
userIds: [DummyValue.UUID],
|
||||
@@ -119,7 +119,6 @@ export class SearchRepository implements ISearchRepository {
|
||||
],
|
||||
})
|
||||
searchDuplicates({ assetId, embedding, maxDistance, type, userIds }: AssetDuplicateSearch) {
|
||||
const vector = asVector(embedding);
|
||||
return this.db
|
||||
.with('cte', (qb) =>
|
||||
qb
|
||||
@@ -127,7 +126,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
.select([
|
||||
'assets.id as assetId',
|
||||
'assets.duplicateId',
|
||||
sql<number>`smart_search.embedding <=> ${vector}`.as('distance'),
|
||||
sql<number>`smart_search.embedding <=> ${embedding}`.as('distance'),
|
||||
])
|
||||
.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
|
||||
.where('assets.ownerId', '=', anyUuid(userIds))
|
||||
@@ -135,7 +134,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
.where('assets.isVisible', '=', true)
|
||||
.where('assets.type', '=', type)
|
||||
.where('assets.id', '!=', asUuid(assetId))
|
||||
.orderBy(sql`smart_search.embedding <=> ${vector}`)
|
||||
.orderBy(sql`smart_search.embedding <=> ${embedding}`)
|
||||
.limit(64),
|
||||
)
|
||||
.selectFrom('cte')
|
||||
@@ -148,7 +147,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
params: [
|
||||
{
|
||||
userIds: [DummyValue.UUID],
|
||||
embedding: Array.from({ length: 512 }, Math.random),
|
||||
embedding: DummyValue.VECTOR,
|
||||
numResults: 10,
|
||||
maxDistance: 0.6,
|
||||
},
|
||||
@@ -159,7 +158,6 @@ export class SearchRepository implements ISearchRepository {
|
||||
throw new Error(`Invalid value for 'numResults': ${numResults}`);
|
||||
}
|
||||
|
||||
const vector = asVector(embedding);
|
||||
return this.db
|
||||
.with('cte', (qb) =>
|
||||
qb
|
||||
@@ -167,14 +165,14 @@ export class SearchRepository implements ISearchRepository {
|
||||
.select([
|
||||
'asset_faces.id',
|
||||
'asset_faces.personId',
|
||||
sql<number>`face_search.embedding <=> ${vector}`.as('distance'),
|
||||
sql<number>`face_search.embedding <=> ${embedding}`.as('distance'),
|
||||
])
|
||||
.innerJoin('assets', 'assets.id', 'asset_faces.assetId')
|
||||
.innerJoin('face_search', 'face_search.faceId', 'asset_faces.id')
|
||||
.where('assets.ownerId', '=', anyUuid(userIds))
|
||||
.where('assets.deletedAt', 'is', null)
|
||||
.$if(!!hasPerson, (qb) => qb.where('asset_faces.personId', 'is not', null))
|
||||
.orderBy(sql`face_search.embedding <=> ${vector}`)
|
||||
.orderBy(sql`face_search.embedding <=> ${embedding}`)
|
||||
.limit(numResults),
|
||||
)
|
||||
.selectFrom('cte')
|
||||
@@ -258,12 +256,11 @@ export class SearchRepository implements ISearchRepository {
|
||||
.execute() as any as Promise<AssetEntity[]>;
|
||||
}
|
||||
|
||||
async upsert(assetId: string, embedding: number[]): Promise<void> {
|
||||
const vector = asVector(embedding);
|
||||
async upsert(assetId: string, embedding: string): Promise<void> {
|
||||
await this.db
|
||||
.insertInto('smart_search')
|
||||
.values({ assetId: asUuid(assetId), embedding: vector } as any)
|
||||
.onConflict((oc) => oc.column('assetId').doUpdateSet({ embedding: vector } as any))
|
||||
.values({ assetId: asUuid(assetId), embedding } as any)
|
||||
.onConflict((oc) => oc.column('assetId').doUpdateSet({ embedding } as any))
|
||||
.execute();
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user