refactor(server)!: move markers and style to dedicated map endpoint/controller (#9832)

* move markers and style to dedicated map endpoint

* chore: open api

* chore: clean up repos

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/repositories/asset.repository.ts

@@ -21,8 +21,6 @@ import {
   AssetUpdateOptions,
   IAssetRepository,
   LivePhotoSearchOptions,
-  MapMarker,
-  MapMarkerSearchOptions,
   MonthDay,
   TimeBucketItem,
   TimeBucketOptions,
@@ -31,7 +29,7 @@ import {
   WithoutProperty,
 } from 'src/interfaces/asset.interface';
 import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
-import { OptionalBetween, searchAssetBuilder } from 'src/utils/database';
+import { searchAssetBuilder } from 'src/utils/database';
 import { Instrumentation } from 'src/utils/instrumentation';
 import { Paginated, PaginationMode, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';
 import {
@@ -547,57 +545,6 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
-  async getMapMarkers(
-    ownerIds: string[],
-    albumIds: string[],
-    options: MapMarkerSearchOptions = {},
-  ): Promise<MapMarker[]> {
-    const { isArchived, isFavorite, fileCreatedAfter, fileCreatedBefore } = options;
-
-    const where = {
-      isVisible: true,
-      isArchived,
-      exifInfo: {
-        latitude: Not(IsNull()),
-        longitude: Not(IsNull()),
-      },
-      isFavorite,
-      fileCreatedAt: OptionalBetween(fileCreatedAfter, fileCreatedBefore),
-    };
-
-    const assets = await this.repository.find({
-      select: {
-        id: true,
-        exifInfo: {
-          city: true,
-          state: true,
-          country: true,
-          latitude: true,
-          longitude: true,
-        },
-      },
-      where: [
-        { ...where, ownerId: In([...ownerIds]) },
-        { ...where, albums: { id: In([...albumIds]) } },
-      ],
-      relations: {
-        exifInfo: true,
-      },
-      order: {
-        fileCreatedAt: 'DESC',
-      },
-    });
-
-    return assets.map((asset) => ({
-      id: asset.id,
-      lat: asset.exifInfo!.latitude!,
-      lon: asset.exifInfo!.longitude!,
-      city: asset.exifInfo!.city,
-      state: asset.exifInfo!.state,
-      country: asset.exifInfo!.country,
-    }));
-  }
-
   async getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats> {
     const builder = this.repository
       .createQueryBuilder('asset')

server/src/repositories/index.ts

@@ -14,6 +14,7 @@ import { IJobRepository } from 'src/interfaces/job.interface';
 import { ILibraryRepository } from 'src/interfaces/library.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { IMachineLearningRepository } from 'src/interfaces/machine-learning.interface';
+import { IMapRepository } from 'src/interfaces/map.interface';
 import { IMediaRepository } from 'src/interfaces/media.interface';
 import { IMemoryRepository } from 'src/interfaces/memory.interface';
 import { IMetadataRepository } from 'src/interfaces/metadata.interface';
@@ -46,6 +47,7 @@ import { JobRepository } from 'src/repositories/job.repository';
 import { LibraryRepository } from 'src/repositories/library.repository';
 import { LoggerRepository } from 'src/repositories/logger.repository';
 import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
+import { MapRepository } from 'src/repositories/map.repository';
 import { MediaRepository } from 'src/repositories/media.repository';
 import { MemoryRepository } from 'src/repositories/memory.repository';
 import { MetadataRepository } from 'src/repositories/metadata.repository';
@@ -64,8 +66,8 @@ import { TagRepository } from 'src/repositories/tag.repository';
 import { UserRepository } from 'src/repositories/user.repository';
 
 export const repositories = [
-  { provide: IActivityRepository, useClass: ActivityRepository },
   { provide: IAccessRepository, useClass: AccessRepository },
+  { provide: IActivityRepository, useClass: ActivityRepository },
   { provide: IAlbumRepository, useClass: AlbumRepository },
   { provide: IAlbumUserRepository, useClass: AlbumUserRepository },
   { provide: IAssetRepository, useClass: AssetRepository },
@@ -76,10 +78,12 @@ export const repositories = [
   { provide: IDatabaseRepository, useClass: DatabaseRepository },
   { provide: IEventRepository, useClass: EventRepository },
   { provide: IJobRepository, useClass: JobRepository },
-  { provide: ILoggerRepository, useClass: LoggerRepository },
-  { provide: ILibraryRepository, useClass: LibraryRepository },
   { provide: IKeyRepository, useClass: ApiKeyRepository },
+  { provide: ILibraryRepository, useClass: LibraryRepository },
+  { provide: ILoggerRepository, useClass: LoggerRepository },
   { provide: IMachineLearningRepository, useClass: MachineLearningRepository },
+  { provide: IMapRepository, useClass: MapRepository },
+  { provide: IMediaRepository, useClass: MediaRepository },
   { provide: IMemoryRepository, useClass: MemoryRepository },
   { provide: IMetadataRepository, useClass: MetadataRepository },
   { provide: IMetricRepository, useClass: MetricRepository },
@@ -87,13 +91,12 @@ export const repositories = [
   { provide: INotificationRepository, useClass: NotificationRepository },
   { provide: IPartnerRepository, useClass: PartnerRepository },
   { provide: IPersonRepository, useClass: PersonRepository },
-  { provide: IServerInfoRepository, useClass: ServerInfoRepository },
-  { provide: ISharedLinkRepository, useClass: SharedLinkRepository },
   { provide: ISearchRepository, useClass: SearchRepository },
+  { provide: IServerInfoRepository, useClass: ServerInfoRepository },
   { provide: ISessionRepository, useClass: SessionRepository },
+  { provide: ISharedLinkRepository, useClass: SharedLinkRepository },
   { provide: IStorageRepository, useClass: StorageRepository },
   { provide: ISystemMetadataRepository, useClass: SystemMetadataRepository },
   { provide: ITagRepository, useClass: TagRepository },
-  { provide: IMediaRepository, useClass: MediaRepository },
   { provide: IUserRepository, useClass: UserRepository },
 ];

server/src/repositories/map.repository.ts

@@ -0,0 +1,246 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+import { getName } from 'i18n-iso-countries';
+import { createReadStream, existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import readLine from 'node:readline';
+import { citiesFile, geodataAdmin1Path, geodataAdmin2Path, geodataCities500Path, geodataDatePath } from 'src/constants';
+import { AssetEntity } from 'src/entities/asset.entity';
+import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
+import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
+import { ILoggerRepository } from 'src/interfaces/logger.interface';
+import {
+  GeoPoint,
+  IMapRepository,
+  MapMarker,
+  MapMarkerSearchOptions,
+  ReverseGeocodeResult,
+} from 'src/interfaces/map.interface';
+import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
+import { OptionalBetween } from 'src/utils/database';
+import { Instrumentation } from 'src/utils/instrumentation';
+import { DataSource, In, IsNull, Not, QueryRunner, Repository } from 'typeorm';
+import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
+
+@Instrumentation()
+@Injectable()
+export class MapRepository implements IMapRepository {
+  constructor(
+    @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
+    @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
+    @InjectDataSource() private dataSource: DataSource,
+    @Inject(ISystemMetadataRepository) private metadataRepository: ISystemMetadataRepository,
+    @Inject(ILoggerRepository) private logger: ILoggerRepository,
+  ) {
+    this.logger.setContext(MapRepository.name);
+  }
+
+  async init(): Promise<void> {
+    this.logger.log('Initializing metadata repository');
+    const geodataDate = await readFile(geodataDatePath, 'utf8');
+
+    // TODO move to service init
+    const geocodingMetadata = await this.metadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
+    if (geocodingMetadata?.lastUpdate === geodataDate) {
+      return;
+    }
+
+    await this.importGeodata();
+
+    await this.metadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
+      lastUpdate: geodataDate,
+      lastImportFileName: citiesFile,
+    });
+
+    this.logger.log('Geodata import completed');
+  }
+
+  async getMapMarkers(
+    ownerIds: string[],
+    albumIds: string[],
+    options: MapMarkerSearchOptions = {},
+  ): Promise<MapMarker[]> {
+    const { isArchived, isFavorite, fileCreatedAfter, fileCreatedBefore } = options;
+
+    const where = {
+      isVisible: true,
+      isArchived,
+      exifInfo: {
+        latitude: Not(IsNull()),
+        longitude: Not(IsNull()),
+      },
+      isFavorite,
+      fileCreatedAt: OptionalBetween(fileCreatedAfter, fileCreatedBefore),
+    };
+
+    const assets = await this.assetRepository.find({
+      select: {
+        id: true,
+        exifInfo: {
+          city: true,
+          state: true,
+          country: true,
+          latitude: true,
+          longitude: true,
+        },
+      },
+      where: [
+        { ...where, ownerId: In([...ownerIds]) },
+        { ...where, albums: { id: In([...albumIds]) } },
+      ],
+      relations: {
+        exifInfo: true,
+      },
+      order: {
+        fileCreatedAt: 'DESC',
+      },
+    });
+
+    return assets.map((asset) => ({
+      id: asset.id,
+      lat: asset.exifInfo!.latitude!,
+      lon: asset.exifInfo!.longitude!,
+      city: asset.exifInfo!.city,
+      state: asset.exifInfo!.state,
+      country: asset.exifInfo!.country,
+    }));
+  }
+
+  async fetchStyle(url: string) {
+    try {
+      const response = await fetch(url);
+
+      if (!response.ok) {
+        throw new Error(`Failed to fetch data from ${url} with status ${response.status}: ${await response.text()}`);
+      }
+
+      return response.json();
+    } catch (error) {
+      throw new Error(`Failed to fetch data from ${url}: ${error}`);
+    }
+  }
+
+  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
+    this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
+
+    const response = await this.geodataPlacesRepository
+      .createQueryBuilder('geoplaces')
+      .where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
+      .orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
+      .limit(1)
+      .getOne();
+
+    if (!response) {
+      this.logger.warn(
+        `Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
+      );
+      return null;
+    }
+
+    this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
+
+    const { countryCode, name: city, admin1Name } = response;
+    const country = getName(countryCode, 'en') ?? null;
+    const state = admin1Name;
+
+    return { country, state, city };
+  }
+
+  private async importGeodata() {
+    const queryRunner = this.dataSource.createQueryRunner();
+    await queryRunner.connect();
+
+    const admin1 = await this.loadAdmin(geodataAdmin1Path);
+    const admin2 = await this.loadAdmin(geodataAdmin2Path);
+
+    try {
+      await queryRunner.startTransaction();
+
+      await queryRunner.manager.clear(GeodataPlacesEntity);
+      await this.loadCities500(queryRunner, admin1, admin2);
+
+      await queryRunner.commitTransaction();
+    } catch (error) {
+      this.logger.fatal('Error importing geodata', error);
+      await queryRunner.rollbackTransaction();
+      throw error;
+    } finally {
+      await queryRunner.release();
+    }
+  }
+
+  private async loadGeodataToTableFromFile(
+    queryRunner: QueryRunner,
+    lineToEntityMapper: (lineSplit: string[]) => GeodataPlacesEntity,
+    filePath: string,
+    options?: { entityFilter?: (linesplit: string[]) => boolean },
+  ) {
+    const _entityFilter = options?.entityFilter ?? (() => true);
+    if (!existsSync(filePath)) {
+      this.logger.error(`Geodata file ${filePath} not found`);
+      throw new Error(`Geodata file ${filePath} not found`);
+    }
+
+    const input = createReadStream(filePath);
+    let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = [];
+    const lineReader = readLine.createInterface({ input });
+
+    for await (const line of lineReader) {
+      const lineSplit = line.split('\t');
+      if (!_entityFilter(lineSplit)) {
+        continue;
+      }
+      const geoData = lineToEntityMapper(lineSplit);
+      bufferGeodata.push(geoData);
+      if (bufferGeodata.length > 1000) {
+        await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
+        bufferGeodata = [];
+      }
+    }
+    await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
+  }
+
+  private async loadCities500(
+    queryRunner: QueryRunner,
+    admin1Map: Map<string, string>,
+    admin2Map: Map<string, string>,
+  ) {
+    await this.loadGeodataToTableFromFile(
+      queryRunner,
+      (lineSplit: string[]) =>
+        this.geodataPlacesRepository.create({
+          id: Number.parseInt(lineSplit[0]),
+          name: lineSplit[1],
+          alternateNames: lineSplit[3],
+          latitude: Number.parseFloat(lineSplit[4]),
+          longitude: Number.parseFloat(lineSplit[5]),
+          countryCode: lineSplit[8],
+          admin1Code: lineSplit[10],
+          admin2Code: lineSplit[11],
+          modificationDate: lineSplit[18],
+          admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`),
+          admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`),
+        }),
+      geodataCities500Path,
+      { entityFilter: (lineSplit) => lineSplit[7] != 'PPLX' },
+    );
+  }
+
+  private async loadAdmin(filePath: string) {
+    if (!existsSync(filePath)) {
+      this.logger.error(`Geodata file ${filePath} not found`);
+      throw new Error(`Geodata file ${filePath} not found`);
+    }
+
+    const input = createReadStream(filePath);
+    const lineReader = readLine.createInterface({ input: input });
+
+    const adminMap = new Map<string, string>();
+    for await (const line of lineReader) {
+      const lineSplit = line.split('\t');
+      adminMap.set(lineSplit[0], lineSplit[1]);
+    }
+
+    return adminMap;
+  }
+}

server/src/repositories/metadata.repository.ts

@@ -2,21 +2,13 @@ import { Inject, Injectable } from '@nestjs/common';
 import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
 import { DefaultReadTaskOptions, Tags, exiftool } from 'exiftool-vendored';
 import geotz from 'geo-tz';
-import { getName } from 'i18n-iso-countries';
-import { createReadStream, existsSync } from 'node:fs';
-import { readFile } from 'node:fs/promises';
-import readLine from 'node:readline';
-import { citiesFile, geodataAdmin1Path, geodataAdmin2Path, geodataCities500Path, geodataDatePath } from 'src/constants';
 import { DummyValue, GenerateSql } from 'src/decorators';
 import { ExifEntity } from 'src/entities/exif.entity';
 import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
-import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
-import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from 'src/interfaces/metadata.interface';
-import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
+import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
 import { Instrumentation } from 'src/utils/instrumentation';
-import { DataSource, QueryRunner, Repository } from 'typeorm';
-import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
+import { DataSource, Repository } from 'typeorm';
 
 @Instrumentation()
 @Injectable()
@@ -24,162 +16,16 @@ export class MetadataRepository implements IMetadataRepository {
   constructor(
     @InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
     @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
-    @Inject(ISystemMetadataRepository)
-    private systemMetadataRepository: ISystemMetadataRepository,
     @InjectDataSource() private dataSource: DataSource,
     @Inject(ILoggerRepository) private logger: ILoggerRepository,
   ) {
     this.logger.setContext(MetadataRepository.name);
   }
 
-  async init(): Promise<void> {
-    this.logger.log('Initializing metadata repository');
-    const geodataDate = await readFile(geodataDatePath, 'utf8');
-
-    // TODO move to metadata service init
-    const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
-    if (geocodingMetadata?.lastUpdate === geodataDate) {
-      return;
-    }
-
-    await this.importGeodata();
-
-    await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
-      lastUpdate: geodataDate,
-      lastImportFileName: citiesFile,
-    });
-
-    this.logger.log('Geodata import completed');
-  }
-
-  private async importGeodata() {
-    const queryRunner = this.dataSource.createQueryRunner();
-    await queryRunner.connect();
-
-    const admin1 = await this.loadAdmin(geodataAdmin1Path);
-    const admin2 = await this.loadAdmin(geodataAdmin2Path);
-
-    try {
-      await queryRunner.startTransaction();
-
-      await queryRunner.manager.clear(GeodataPlacesEntity);
-      await this.loadCities500(queryRunner, admin1, admin2);
-
-      await queryRunner.commitTransaction();
-    } catch (error) {
-      this.logger.fatal('Error importing geodata', error);
-      await queryRunner.rollbackTransaction();
-      throw error;
-    } finally {
-      await queryRunner.release();
-    }
-  }
-
-  private async loadGeodataToTableFromFile(
-    queryRunner: QueryRunner,
-    lineToEntityMapper: (lineSplit: string[]) => GeodataPlacesEntity,
-    filePath: string,
-    options?: { entityFilter?: (linesplit: string[]) => boolean },
-  ) {
-    const _entityFilter = options?.entityFilter ?? (() => true);
-    if (!existsSync(filePath)) {
-      this.logger.error(`Geodata file ${filePath} not found`);
-      throw new Error(`Geodata file ${filePath} not found`);
-    }
-
-    const input = createReadStream(filePath);
-    let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = [];
-    const lineReader = readLine.createInterface({ input });
-
-    for await (const line of lineReader) {
-      const lineSplit = line.split('\t');
-      if (!_entityFilter(lineSplit)) {
-        continue;
-      }
-      const geoData = lineToEntityMapper(lineSplit);
-      bufferGeodata.push(geoData);
-      if (bufferGeodata.length > 1000) {
-        await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
-        bufferGeodata = [];
-      }
-    }
-    await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
-  }
-
-  private async loadCities500(
-    queryRunner: QueryRunner,
-    admin1Map: Map<string, string>,
-    admin2Map: Map<string, string>,
-  ) {
-    await this.loadGeodataToTableFromFile(
-      queryRunner,
-      (lineSplit: string[]) =>
-        this.geodataPlacesRepository.create({
-          id: Number.parseInt(lineSplit[0]),
-          name: lineSplit[1],
-          alternateNames: lineSplit[3],
-          latitude: Number.parseFloat(lineSplit[4]),
-          longitude: Number.parseFloat(lineSplit[5]),
-          countryCode: lineSplit[8],
-          admin1Code: lineSplit[10],
-          admin2Code: lineSplit[11],
-          modificationDate: lineSplit[18],
-          admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`),
-          admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`),
-        }),
-      geodataCities500Path,
-      { entityFilter: (lineSplit) => lineSplit[7] != 'PPLX' },
-    );
-  }
-
-  private async loadAdmin(filePath: string) {
-    if (!existsSync(filePath)) {
-      this.logger.error(`Geodata file ${filePath} not found`);
-      throw new Error(`Geodata file ${filePath} not found`);
-    }
-
-    const input = createReadStream(filePath);
-    const lineReader = readLine.createInterface({ input: input });
-
-    const adminMap = new Map<string, string>();
-    for await (const line of lineReader) {
-      const lineSplit = line.split('\t');
-      adminMap.set(lineSplit[0], lineSplit[1]);
-    }
-
-    return adminMap;
-  }
-
   async teardown() {
     await exiftool.end();
   }
 
-  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
-    this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
-
-    const response = await this.geodataPlacesRepository
-      .createQueryBuilder('geoplaces')
-      .where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
-      .orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
-      .limit(1)
-      .getOne();
-
-    if (!response) {
-      this.logger.warn(
-        `Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
-      );
-      return null;
-    }
-
-    this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
-
-    const { countryCode, name: city, admin1Name } = response;
-    const country = getName(countryCode, 'en') ?? null;
-    const state = admin1Name;
-
-    return { country, state, city };
-  }
-
   readTags(path: string): Promise<ImmichTags | null> {
     return exiftool
       .read(path, undefined, {

server/src/repositories/system-metadata.repository.ts

@@ -26,20 +26,6 @@ export class SystemMetadataRepository implements ISystemMetadataRepository {
     await this.repository.upsert({ key, value }, { conflictPaths: { key: true } });
   }
 
-  async fetchStyle(url: string) {
-    try {
-      const response = await fetch(url);
-
-      if (!response.ok) {
-        throw new Error(`Failed to fetch data from ${url} with status ${response.status}: ${await response.text()}`);
-      }
-
-      return response.json();
-    } catch (error) {
-      throw new Error(`Failed to fetch data from ${url}: ${error}`);
-    }
-  }
-
   readFile(filename: string): Promise<string> {
     return readFile(filename, { encoding: 'utf8' });
   }