fix(server): don't crash when refreshing large libraries (#7934)

* add job to check for offline files

* fix lint

* only check for offline when using checkForOffline

* improve tests

* remove old test

* wip

* remove trie

* refactor batches

* also check offline status

* fix spelling

* don't do offline scan

* rename scan to check

* fix job statuses

* fix lint

* cleanup

* add test

* open-api

* fix test

* fix spinner

* reset text

* don't double batch

* fix comments from mert

* remove tries

* fix tests

* fix e2e

* fix test

* fix test

* add tests

* fix lint

* fix e2e

* interweave scans

* fix errors

* fix messages

* fix test

* add mock

* fix sql

* fix e2e

* use library batch size

* save -> update

* add file extensions

* update specs

* test for import paths

* check import paths when testing offline

* fix lint

* normalize import path

* remove console logs

* decrease batch size to 1000

* add test for import path

* add test for already-online assets

* fix merge

* fix lint

* add library job back

* add offline job to correct queue

* library spec compiles now

* move one test to new e2e

* fix comments

* fix comments

* fix lint

* refactor path validation

* fix loop bug

* remove logging

* expect responses

* fix asset mock

* take the straightforward approach

* use generator correctly

* fix vitest on file edit

* bump vitest to 1.6.0

* test for offline check

* add e2e tests for offlining assets depending on import path

* cleanup e2e test after finish

* cleanup library service

* paginate the walk generator

* fix tests

* fix typo

* refactoring handleOfflineCheck

* better testing of handleOfflineCheck

* fix lint

* handle large library deletions

* dont check if library is deleted

* fix mock

* add a 100k page size to library

* fix loading animation

* better log messages

* Better logging for offline asset removal

* fix sql and tests

* fix number format

* Remove submodule

* fix format

* chore: cleanup

* chore: fix tests

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>

server/src/repositories/asset.repository.ts

@@ -383,7 +383,7 @@ export class AssetRepository implements IAssetRepository {
 
   @GenerateSql(
     ...Object.values(WithProperty)
-      .filter((property) => property !== WithProperty.IS_OFFLINE)
+      .filter((property) => property !== WithProperty.IS_OFFLINE && property !== WithProperty.IS_ONLINE)
       .map((property) => ({
         name: property,
         params: [DummyValue.PAGINATION, property],
@@ -539,7 +539,14 @@ export class AssetRepository implements IAssetRepository {
         if (!libraryId) {
           throw new Error('Library id is required when finding offline assets');
         }
-        where = [{ isOffline: true, libraryId: libraryId }];
+        where = [{ isOffline: true, libraryId }];
+        break;
+      }
+      case WithProperty.IS_ONLINE: {
+        if (!libraryId) {
+          throw new Error('Library id is required when finding online assets');
+        }
+        where = [{ isOffline: false, libraryId }];
         break;
       }
 

server/src/repositories/job.repository.ts

@@ -79,6 +79,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY,
   [JobName.LIBRARY_SCAN]: QueueName.LIBRARY,
   [JobName.LIBRARY_DELETE]: QueueName.LIBRARY,
+  [JobName.LIBRARY_CHECK_OFFLINE]: QueueName.LIBRARY,
   [JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY,
   [JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY,
   [JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY,

server/src/repositories/library.repository.ts

@@ -94,30 +94,6 @@ export class LibraryRepository implements ILibraryRepository {
     };
   }
 
-  @GenerateSql({ params: [DummyValue.UUID] })
-  async getAssetIds(libraryId: string, withDeleted = false): Promise<string[]> {
-    const builder = this.repository
-      .createQueryBuilder('library')
-      .innerJoinAndSelect('library.assets', 'assets')
-      .where('library.id = :id', { id: libraryId })
-      .select('assets.id');
-
-    if (withDeleted) {
-      builder.withDeleted();
-    }
-
-    // Return all asset paths for a given library
-    const rawResults = await builder.getRawMany();
-
-    const results: string[] = [];
-
-    for (const rawPath of rawResults) {
-      results.push(rawPath.assets_id);
-    }
-
-    return results;
-  }
-
   private async save(library: Partial<LibraryEntity>) {
     const { id } = await this.repository.save(library);
     return this.repository.findOneByOrFail({ id });

server/src/repositories/storage.repository.ts

@@ -5,7 +5,7 @@ import { escapePath, glob, globStream } from 'fast-glob';
 import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import { CrawlOptionsDto } from 'src/dtos/library.dto';
+import { CrawlOptionsDto, WalkOptionsDto } from 'src/dtos/library.dto';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import {
   DiskUsage,
@@ -157,8 +157,8 @@ export class StorageRepository implements IStorageRepository {
     });
   }
 
-  async *walk(crawlOptions: CrawlOptionsDto): AsyncGenerator<string> {
-    const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
+  async *walk(walkOptions: WalkOptionsDto): AsyncGenerator<string[]> {
+    const { pathsToCrawl, exclusionPatterns, includeHidden } = walkOptions;
     if (pathsToCrawl.length === 0) {
       async function* emptyGenerator() {}
       return emptyGenerator();
@@ -172,8 +172,17 @@ export class StorageRepository implements IStorageRepository {
       ignore: exclusionPatterns,
     });
 
+    let batch: string[] = [];
     for await (const value of stream) {
-      yield value as string;
+      batch.push(value.toString());
+      if (batch.length === walkOptions.take) {
+        yield batch;
+        batch = [];
+      }
+    }
+
+    if (batch.length > 0) {
+      yield batch;
     }
   }