fix: development containers init race conditions

docker/docker-compose.dev.yml

@@ -14,33 +14,65 @@
 name: immich-dev
 
 services:
+  immich-app-base:
+    profiles: ['_base']
+    tmpfs:
+      - /tmp
+    volumes:
+      - ..:/usr/src/app
+      - pnpm_cache:/buildcache/pnpm_cache
+      - server_node_modules:/usr/src/app/server/node_modules
+      - web_node_modules:/usr/src/app/web/node_modules
+      - github_node_modules:/usr/src/app/.github/node_modules
+      - cli_node_modules:/usr/src/app/cli/node_modules
+      - docs_node_modules:/usr/src/app/docs/node_modules
+      - e2e_node_modules:/usr/src/app/e2e/node_modules
+      - sdk_node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
+      - app_node_modules:/usr/src/app/node_modules
+      - sveltekit:/usr/src/app/web/.svelte-kit
+      - coverage:/usr/src/app/web/coverage
+
+  immich-init:
+    extends:
+      service: immich-app-base
+    profiles: !reset []
+    container_name: immich_init
+    image: immich-server-dev:latest
+    build:
+      context: ../
+      dockerfile: server/Dockerfile.dev
+      target: dev
+    command:
+      - |
+        pnpm install
+        touch /tmp/init-complete
+        exec tail -f /dev/null
+    volumes:
+      - pnpm_store_server:/buildcache/pnpm-store
+    restart: 'no'
+    healthcheck:
+      test: ['CMD', 'test', '-f', '/tmp/init-complete']
+      interval: 2s
+      timeout: 3s
+      retries: 300
+      start_period: 300s
+
   immich-server:
+    extends:
+      service: immich-app-base
+    profiles: !reset []
     container_name: immich_server
     command: ['immich-dev']
     image: immich-server-dev:latest
-    # extends:
-    #   file: hwaccel.transcoding.yml
-    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
     build:
       context: ../
       dockerfile: server/Dockerfile.dev
       target: dev
     restart: unless-stopped
     volumes:
-      - ..:/usr/src/app
       - ${UPLOAD_LOCATION}/photos:/data
       - /etc/localtime:/etc/localtime:ro
-      - pnpm-store:/usr/src/app/.pnpm-store
-      - server-node_modules:/usr/src/app/server/node_modules
-      - web-node_modules:/usr/src/app/web/node_modules
-      - github-node_modules:/usr/src/app/.github/node_modules
-      - cli-node_modules:/usr/src/app/cli/node_modules
-      - docs-node_modules:/usr/src/app/docs/node_modules
-      - e2e-node_modules:/usr/src/app/e2e/node_modules
-      - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
-      - app-node_modules:/usr/src/app/node_modules
-      - sveltekit:/usr/src/app/web/.svelte-kit
-      - coverage:/usr/src/app/web/coverage
+      - pnpm_store_server:/buildcache/pnpm-store
       - ../plugins:/build/corePlugin
     env_file:
       - .env
@@ -63,6 +95,8 @@ services:
       - 9231:9231
       - 2283:2283
     depends_on:
+      immich-init:
+        condition: service_healthy
       redis:
         condition: service_started
       database:
@@ -71,6 +105,9 @@ services:
       disable: false
 
   immich-web:
+    extends:
+      service: immich-app-base
+    profiles: !reset []
     container_name: immich_web
     image: immich-web-dev:latest
     build:
@@ -84,20 +121,11 @@ services:
       - 3000:3000
       - 24678:24678
     volumes:
-      - ..:/usr/src/app
-      - pnpm-store:/usr/src/app/.pnpm-store
-      - server-node_modules:/usr/src/app/server/node_modules
-      - web-node_modules:/usr/src/app/web/node_modules
-      - github-node_modules:/usr/src/app/.github/node_modules
-      - cli-node_modules:/usr/src/app/cli/node_modules
-      - docs-node_modules:/usr/src/app/docs/node_modules
-      - e2e-node_modules:/usr/src/app/e2e/node_modules
-      - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
-      - app-node_modules:/usr/src/app/node_modules
-      - sveltekit:/usr/src/app/web/.svelte-kit
-      - coverage:/usr/src/app/web/coverage
+      - pnpm_store_web:/buildcache/pnpm-store
     restart: unless-stopped
     depends_on:
+      immich-init:
+        condition: service_healthy
       immich-server:
         condition: service_started
 
@@ -116,7 +144,7 @@ services:
       - 3003:3003
     volumes:
       - ../machine-learning/immich_ml:/usr/src/immich_ml
-      - model-cache:/cache
+      - model_cache:/cache
     env_file:
       - .env
     depends_on:
@@ -156,7 +184,7 @@ services:
   #   image: prom/prometheus
   #   volumes:
   #     - ./prometheus.yml:/etc/prometheus/prometheus.yml
-  #     - prometheus-data:/prometheus
+  #     - prometheus_data:/prometheus
 
   # first login uses admin/admin
   # add data source for http://immich-prometheus:9090 to get started
@@ -167,20 +195,22 @@ services:
   #     - 3000:3000
   #   image: grafana/grafana:10.3.3-ubuntu
   #   volumes:
-  #     - grafana-data:/var/lib/grafana
+  #     - grafana_data:/var/lib/grafana
 
 volumes:
-  model-cache:
-  prometheus-data:
-  grafana-data:
-  pnpm-store:
-  server-node_modules:
-  web-node_modules:
-  github-node_modules:
-  cli-node_modules:
-  docs-node_modules:
-  e2e-node_modules:
-  sdk-node_modules:
-  app-node_modules:
+  model_cache:
+  prometheus_data:
+  grafana_data:
+  pnpm_cache:
+  pnpm_store_server:
+  pnpm_store_web:
+  server_node_modules:
+  web_node_modules:
+  github_node_modules:
+  cli_node_modules:
+  docs_node_modules:
+  e2e_node_modules:
+  sdk_node_modules:
+  app_node_modules:
   sveltekit:
   coverage:

e2e/docker-compose.dev.yml

@@ -1,86 +1,77 @@
 name: immich-e2e
 
 services:
+  immich-app-base:
+    extends:
+      file: ../docker/docker-compose.dev.yml
+      service: immich-app-base
+
+  immich-init:
+    extends:
+      file: ../docker/docker-compose.dev.yml
+      service: immich-init
+    container_name: immich-e2e-init
+
   immich-server:
+    extends:
+      file: ../docker/docker-compose.dev.yml
+      service: immich-server
     container_name: immich-e2e-server
-    command: ['immich-dev']
-    image: immich-server-dev:latest
-    build:
-      context: ../
-      dockerfile: server/Dockerfile.dev
-      target: dev
+    ports: !reset []
+    env_file: !reset []
     environment:
-      - DB_HOSTNAME=database
-      - DB_USERNAME=postgres
-      - DB_PASSWORD=postgres
-      - DB_DATABASE_NAME=immich
-      - IMMICH_MACHINE_LEARNING_ENABLED=false
-      - IMMICH_TELEMETRY_INCLUDE=all
-      - IMMICH_ENV=testing
-      - IMMICH_PORT=2285
-      - IMMICH_IGNORE_MOUNT_CHECK_ERRORS=true
+      DB_HOSTNAME: database
+      DB_USERNAME: postgres
+      DB_PASSWORD: postgres
+      DB_DATABASE_NAME: immich
+      IMMICH_MACHINE_LEARNING_ENABLED: 'false'
+      IMMICH_TELEMETRY_INCLUDE: all
+      IMMICH_ENV: testing
+      IMMICH_PORT: '2285'
+      IMMICH_IGNORE_MOUNT_CHECK_ERRORS: 'true'
     volumes:
       - ./test-assets:/test-assets
-      - ..:/usr/src/app
-      - ${UPLOAD_LOCATION}/photos:/data
-      - /etc/localtime:/etc/localtime:ro
-      - pnpm-store:/usr/src/app/.pnpm-store
-      - server-node_modules:/usr/src/app/server/node_modules
-      - web-node_modules:/usr/src/app/web/node_modules
-      - github-node_modules:/usr/src/app/.github/node_modules
-      - cli-node_modules:/usr/src/app/cli/node_modules
-      - docs-node_modules:/usr/src/app/docs/node_modules
-      - e2e-node_modules:/usr/src/app/e2e/node_modules
-      - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
-      - app-node_modules:/usr/src/app/node_modules
-      - sveltekit:/usr/src/app/web/.svelte-kit
-      - coverage:/usr/src/app/web/coverage
-      - ../plugins:/build/corePlugin
     depends_on:
+      immich-init:
+        condition: service_healthy
       redis:
         condition: service_started
       database:
         condition: service_healthy
 
   immich-web:
+    extends:
+      file: ../docker/docker-compose.dev.yml
+      service: immich-web
     container_name: immich-e2e-web
-    image: immich-web-dev:latest
-    build:
-      context: ../
-      dockerfile: server/Dockerfile.dev
-      target: dev
-    command: ['immich-web']
-    ports:
+    ports: !override
       - 2285:3000
     environment:
-      - IMMICH_SERVER_URL=http://immich-server:2285/
-    volumes:
-      - ..:/usr/src/app
-      - pnpm-store:/usr/src/app/.pnpm-store
-      - server-node_modules:/usr/src/app/server/node_modules
-      - web-node_modules:/usr/src/app/web/node_modules
-      - github-node_modules:/usr/src/app/.github/node_modules
-      - cli-node_modules:/usr/src/app/cli/node_modules
-      - docs-node_modules:/usr/src/app/docs/node_modules
-      - e2e-node_modules:/usr/src/app/e2e/node_modules
-      - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
-      - app-node_modules:/usr/src/app/node_modules
-      - sveltekit:/usr/src/app/web/.svelte-kit
-      - coverage:/usr/src/app/web/coverage
+      IMMICH_SERVER_URL: http://immich-server:2285/
+    depends_on:
+      immich-init:
+        condition: service_healthy
     restart: unless-stopped
 
   redis:
-    image: redis:6.2-alpine@sha256:46884be93652d02a96a176ccf173d1040bef365c5706aa7b6a1931caec8bfeef
+    extends:
+      file: ../docker/docker-compose.dev.yml
+      service: redis
+    container_name: immich-e2e-redis
 
   database:
-    image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0@sha256:6f3e9d2c2177af16c2988ff71425d79d89ca630ec2f9c8db03209ab716542338
+    extends:
+      file: ../docker/docker-compose.dev.yml
+      service: database
+    container_name: immich-e2e-postgres
     command: -c fsync=off -c shared_preload_libraries=vchord.so -c config_file=/var/lib/postgresql/data/postgresql.conf
+    env_file: !reset []
+    ports: !override
+      - 5435:5432
     environment:
       POSTGRES_PASSWORD: postgres
       POSTGRES_USER: postgres
       POSTGRES_DB: immich
-    ports:
-      - 5435:5432
     healthcheck:
       test: ['CMD-SHELL', 'pg_isready -U postgres -d immich']
       interval: 1s
@@ -89,17 +80,19 @@ services:
       start_period: 10s
 
 volumes:
-  model-cache:
-  prometheus-data:
-  grafana-data:
-  pnpm-store:
-  server-node_modules:
-  web-node_modules:
-  github-node_modules:
-  cli-node_modules:
-  docs-node_modules:
-  e2e-node_modules:
-  sdk-node_modules:
-  app-node_modules:
+  model_cache:
+  prometheus_data:
+  grafana_data:
+  pnpm_cache:
+  pnpm_store_server:
+  pnpm_store_web:
+  server_node_modules:
+  web_node_modules:
+  github_node_modules:
+  cli_node_modules:
+  docs_node_modules:
+  e2e_node_modules:
+  sdk_node_modules:
+  app_node_modules:
   sveltekit:
   coverage:

e2e/docker-compose.yml

@@ -2,6 +2,7 @@ name: immich-e2e
 
 services:
   e2e-auth-server:
+    container_name: immich-e2e-auth-server
     build:
       context: ../e2e-auth-server
     ports:
@@ -22,15 +23,15 @@ services:
         - BUILD_SOURCE_REF=e2e
         - BUILD_SOURCE_COMMIT=e2eeeeeeeeeeeeeeeeee
     environment:
-      - DB_HOSTNAME=database
-      - DB_USERNAME=postgres
-      - DB_PASSWORD=postgres
-      - DB_DATABASE_NAME=immich
-      - IMMICH_MACHINE_LEARNING_ENABLED=false
-      - IMMICH_TELEMETRY_INCLUDE=all
-      - IMMICH_ENV=testing
-      - IMMICH_PORT=2285
-      - IMMICH_IGNORE_MOUNT_CHECK_ERRORS=true
+      DB_HOSTNAME: database
+      DB_USERNAME: postgres
+      DB_PASSWORD: postgres
+      DB_DATABASE_NAME: immich
+      IMMICH_MACHINE_LEARNING_ENABLED: 'false'
+      IMMICH_TELEMETRY_INCLUDE: all
+      IMMICH_ENV: testing
+      IMMICH_PORT: '2285'
+      IMMICH_IGNORE_MOUNT_CHECK_ERRORS: 'true'
     volumes:
       - ./test-assets:/test-assets
     depends_on:
@@ -42,10 +43,14 @@ services:
       - 2285:2285
 
   redis:
-    image: redis:6.2-alpine@sha256:46884be93652d02a96a176ccf173d1040bef365c5706aa7b6a1931caec8bfeef
+    container_name: immich-e2e-redis
+    image: docker.io/valkey/valkey:9@sha256:546304417feac0874c3dd576e0952c6bb8f06bb4093ea0c9ca303c73cf458f63
+    healthcheck:
+      test: redis-cli ping || exit 1
 
   database:
-    image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0@sha256:6f3e9d2c2177af16c2988ff71425d79d89ca630ec2f9c8db03209ab716542338
+    container_name: immich-e2e-postgres
+    image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23
     command: -c fsync=off -c shared_preload_libraries=vchord.so -c config_file=/var/lib/postgresql/data/postgresql.conf
     environment:
       POSTGRES_PASSWORD: postgres

server/Dockerfile.dev

@@ -3,19 +3,19 @@ FROM ghcr.io/immich-app/base-server-dev:202601131104@sha256:8d907eb3fe10dba4a1e0
 
 ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
   CI=1 \
-  COREPACK_HOME=/tmp
+  COREPACK_HOME=/tmp \ 
+  PNPM_HOME=/buildcache/pnpm-store
 
 RUN npm install --global corepack@latest && \
   corepack enable pnpm && \
+  echo "devdir=/buildcache/node-gyp" >> /usr/local/etc/npmrc && \
   echo "store-dir=/buildcache/pnpm-store" >> /usr/local/etc/npmrc && \
-  echo "devdir=/buildcache/node-gyp" >> /usr/local/etc/npmrc
+  echo "cache-dir=/buildcache/pnpm-cache" >> /usr/local/etc/npmrc && \
+  echo "# Retry configuration - default is 2" >> /usr/local/etc/npmrc && \
+  echo "fetch-retries=5" >> /usr/local/etc/npmrc && \
+  mkdir -p /buildcache/pnpm-store /buildcache/pnpm-cache /buildcache/node-gyp && \
+  chmod -R o+rw /buildcache
 
-COPY ./package* ./pnpm* .pnpmfile.cjs /tmp/create-dep-cache/
-COPY ./web/package* ./web/pnpm* /tmp/create-dep-cache/web/
-COPY ./server/package* ./server/pnpm* /tmp/create-dep-cache/server/
-COPY ./open-api/typescript-sdk/package* ./open-api/typescript-sdk/pnpm* /tmp/create-dep-cache/open-api/typescript-sdk/
-WORKDIR /tmp/create-dep-cache
-RUN pnpm fetch && rm -rf /tmp/create-dep-cache && chmod -R o+rw /buildcache
 WORKDIR /usr/src/app
 
 ENV PATH="${PATH}:/usr/src/app/server/bin:/usr/src/app/web/bin" \