feat(ml): enable openvino for cpu (#22948)

* Enable OpenVINO CPU acceleration in Immich

* Remove unnecessary debug log

* Removing checking for device_ids for openvino since cpu will always be available

* Find OpenVINOExecutionProvider index instead of assuming index 0

* Fix openvino tests

* Fix failing test mock. OpenVINO expects provider options, but cuda provide doesn't so use that for mocked tests.

* Support empty provider options in OrtSessions in which case ONNXRuntime will use its own defaults

* Use OpenVINOExecutionProvider for test_sets_provider_options_kwarg

* fix mock

* simplify

* unused variable

---------

Co-authored-by: Aleksander <pejcic@adobe.com>
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

machine-learning/immich_ml/sessions/ort.py

@@ -64,14 +64,6 @@ class OrtSession:
     def _providers_default(self) -> list[str]:
         available_providers = set(ort.get_available_providers())
         log.debug(f"Available ORT providers: {available_providers}")
-        if (openvino := "OpenVINOExecutionProvider") in available_providers:
-            device_ids: list[str] = ort.capi._pybind_state.get_available_openvino_device_ids()
-            log.debug(f"Available OpenVINO devices: {device_ids}")
-
-            gpu_devices = [device_id for device_id in device_ids if device_id.startswith("GPU")]
-            if not gpu_devices:
-                log.warning("No GPU device found in OpenVINO. Falling back to CPU.")
-                available_providers.remove(openvino)
         return [provider for provider in SUPPORTED_PROVIDERS if provider in available_providers]
 
     @property
@@ -102,12 +94,19 @@ class OrtSession:
                         "migraphx_fp16_enable": "1" if settings.rocm_precision == ModelPrecision.FP16 else "0",
                     }
                 case "OpenVINOExecutionProvider":
-                    openvino_dir = self.model_path.parent / "openvino"
+                    device_ids: list[str] = ort.capi._pybind_state.get_available_openvino_device_ids()
-                    device = f"GPU.{settings.device_id}"
+                    # Check for available devices, preferring GPU over CPU
+                    gpu_devices = [d for d in device_ids if d.startswith("GPU")]
+                    if gpu_devices:
+                        device_type = f"GPU.{settings.device_id}"
+                        log.debug(f"OpenVINO: Using GPU device {device_type}")
+                    else:
+                        device_type = "CPU"
+                        log.debug("OpenVINO: No GPU found, using CPU")
                     options = {
-                        "device_type": device,
+                        "device_type": device_type,
                         "precision": settings.openvino_precision.value,
-                        "cache_dir": openvino_dir.as_posix(),
+                        "cache_dir": (self.model_path.parent / "openvino").as_posix(),
                     }
                 case "CoreMLExecutionProvider":
                     options = {
@@ -139,12 +138,14 @@ class OrtSession:
         sess_options.enable_cpu_mem_arena = settings.model_arena
 
         # avoid thread contention between models
+        # Set inter_op threads
         if settings.model_inter_op_threads > 0:
             sess_options.inter_op_num_threads = settings.model_inter_op_threads
         # these defaults work well for CPU, but bottleneck GPU
         elif settings.model_inter_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:
             sess_options.inter_op_num_threads = 1
 
+        # Set intra_op threads
         if settings.model_intra_op_threads > 0:
             sess_options.intra_op_num_threads = settings.model_intra_op_threads
         elif settings.model_intra_op_threads == 0 and self.providers == ["CPUExecutionProvider"]:

machine-learning/test_main.py

@@ -204,13 +204,6 @@ class TestOrtSession:
 
         assert session.providers == self.OV_EP
 
-    @pytest.mark.ov_device_ids(["CPU"])
-    @pytest.mark.providers(OV_EP)
-    def test_avoids_openvino_if_gpu_not_available(self, providers: list[str], ov_device_ids: list[str]) -> None:
-        session = OrtSession("ViT-B-32__openai")
-
-        assert session.providers == self.CPU_EP
-
     @pytest.mark.providers(CUDA_EP_OUT_OF_ORDER)
     def test_sets_providers_in_correct_order(self, providers: list[str]) -> None:
         session = OrtSession("ViT-B-32__openai")
@@ -256,7 +249,8 @@ class TestOrtSession:
             {"arena_extend_strategy": "kSameAsRequested"},
         ]
 
-    def test_sets_provider_options_for_openvino(self) -> None:
+    @pytest.mark.ov_device_ids(["GPU.0", "GPU.1", "CPU"])
+    def test_sets_provider_options_for_openvino(self, ov_device_ids: list[str]) -> None:
         model_path = "/cache/ViT-B-32__openai/textual/model.onnx"
         os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"
 
@@ -270,7 +264,8 @@ class TestOrtSession:
             }
         ]
 
-    def test_sets_openvino_to_fp16_if_enabled(self, mocker: MockerFixture) -> None:
+    @pytest.mark.ov_device_ids(["GPU.0", "GPU.1", "CPU"])
+    def test_sets_openvino_to_fp16_if_enabled(self, ov_device_ids: list[str], mocker: MockerFixture) -> None:
         model_path = "/cache/ViT-B-32__openai/textual/model.onnx"
         os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"
         mocker.patch.object(settings, "openvino_precision", ModelPrecision.FP16)
@@ -285,6 +280,19 @@ class TestOrtSession:
             }
         ]
 
+    @pytest.mark.ov_device_ids(["CPU"])
+    def test_sets_provider_options_for_openvino_cpu(self, ov_device_ids: list[str]) -> None:
+        model_path = "/cache/ViT-B-32__openai/model.onnx"
+        session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])
+
+        assert session.provider_options == [
+            {
+                "device_type": "CPU",
+                "precision": "FP32",
+                "cache_dir": "/cache/ViT-B-32__openai/openvino",
+            }
+        ]
+
     def test_sets_provider_options_for_cuda(self) -> None:
         os.environ["MACHINE_LEARNING_DEVICE_ID"] = "1"
 
@@ -341,6 +349,23 @@ class TestOrtSession:
         assert session.sess_options.inter_op_num_threads == 1
         assert session.sess_options.intra_op_num_threads == 2
 
+    @pytest.mark.ov_device_ids(["CPU"])
+    def test_sets_default_sess_options_if_openvino_cpu(self, ov_device_ids: list[str]) -> None:
+        model_path = "/cache/ViT-B-32__openai/model.onnx"
+        session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])
+
+        assert session.sess_options.execution_mode == ort.ExecutionMode.ORT_SEQUENTIAL
+        assert session.sess_options.inter_op_num_threads == 0
+        assert session.sess_options.intra_op_num_threads == 0
+
+    @pytest.mark.ov_device_ids(["GPU.0", "CPU"])
+    def test_sets_default_sess_options_if_openvino_gpu(self, ov_device_ids: list[str]) -> None:
+        model_path = "/cache/ViT-B-32__openai/model.onnx"
+        session = OrtSession(model_path, providers=["OpenVINOExecutionProvider"])
+
+        assert session.sess_options.inter_op_num_threads == 0
+        assert session.sess_options.intra_op_num_threads == 0
+
     def test_sets_default_sess_options_does_not_set_threads_if_non_cpu_and_default_threads(self) -> None:
         session = OrtSession("ViT-B-32__openai", providers=["CUDAExecutionProvider", "CPUExecutionProvider"])