diff --git a/BUILDING.md b/BUILDING.md index 0820aab..0a16903 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -44,4 +44,37 @@ You also need to have CMake installed. You can obtain this using homebrew: brew install cmake ``` -CMake can also be installed from https://cmake.org/download/ but `cmake` binary needs to be in your PATH. \ No newline at end of file +CMake can also be installed from https://cmake.org/download/ but `cmake` binary needs to be in your PATH. + +# OpenVINO support + +## Development Tools +OpenVINO support requires the OpenVINO Development Tools to be installed. You can find +instructions for installing the OpenVINO Development Tools here: +https://docs.openvino.ai/2023.0/openvino_docs_install_guides_install_dev_tools.html#for-c-developers + +On Arch Linux, you can install the OpenVINO Development Tools with the following command: +``` +paru -S openvino +``` +This build may take a significant amount of time, but can save massive headaches later on. + +## Building +First, the `openvino` feature must be enabled in your Cargo.toml. + +Next, you must set the `OpenVINO_DIR` environment variable to the path where CMake can find +`OpenVINOConfig.cmake`. +This is usually in the `cmake` directory of the OpenVINO installation. + +If you used the AUR package to install OpenVINO, the location of this file is `/opt/intel/openvino/runtime/cmake`. + +``` +export OpenVINO_DIR=/opt/intel/openvino/runtime/cmake +``` + +Finally, you can build whisper-rs as normal. + +## Tested platforms +- Arch Linux + +If you have successfully built whisper-rs with OpenVINO on another platform, please open a PR to document it here! \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 85cabd7..ffeff0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ simd = [] coreml = ["whisper-rs-sys/coreml"] cuda = ["whisper-rs-sys/cuda"] opencl = ["whisper-rs-sys/opencl"] +openvino = ["whisper-rs-sys/openvino"] test-with-tiny-model = [] [package.metadata.docs.rs] diff --git a/README.md b/README.md index 49ab481..f772538 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,9 @@ See the docs: https://docs.rs/whisper-rs/ for more details. ## Building -See [BUILDING.md](BUILDING.md) for instructions for building whisper-rs on Windows and OSX M1. Linux builds should just work out of the box. +See [BUILDING.md](BUILDING.md) for instructions for building whisper-rs on Windows and OSX M1, +or with OpenVINO on any OS. +Besides OpenVINO, Linux builds should just work out of the box. ## Troubleshooting diff --git a/src/whisper_ctx.rs b/src/whisper_ctx.rs index e035c07..29f3685 100644 --- a/src/whisper_ctx.rs +++ b/src/whisper_ctx.rs @@ -56,6 +56,44 @@ impl WhisperContext { // we don't implement `whisper_init()` here since i have zero clue what `whisper_model_loader` does + /// Using this context, enable use of OpenVINO for encoder inference. + /// + /// # Arguments + /// * `model_path`: An optional path to the OpenVINO encoder IR model. + /// If set to `None`, + /// the path will be generated from the ggml model path + /// that was passed in to whisper_init_from_file. + /// For example, if the model path was "/path/to/ggml-base.en.bin", + /// then the OpenVINO IR model path will be assumed as "/path/to/ggml-base.en-encoder-openvino.xml". + /// + /// * `device`: The OpenVINO device to use for inference (e.g. "CPU", "GPU") + /// + /// * `cache_dir`: Optional cache directory that can speed up init time, + /// especially for GPU, by caching compiled 'blobs' there. + /// Set to nullptr if not used. + /// + /// # Returns + /// `true` on success, `false` if OpenVINO was not enabled at compile time + /// (enable the `openvino` feature flag in your Cargo.toml). + /// + /// # C++ equivalent + /// `int whisper_ctx_init_openvino_encoder(struct whisper_context * ctx, const char * model_path, const char * device, const char * cache_dir);` + #[cfg(feature = "openvino")] + pub fn init_openvino_encoder(&mut self, model_path: Option<&str>, device: &str, cache_dir: Option<&str>) -> bool { + let model_path = model_path.map(|s| CString::new(s).unwrap()); + let device = CString::new(device).unwrap(); + let cache_dir = cache_dir.map(|s| CString::new(s).unwrap()); + let ret = unsafe { + whisper_rs_sys::whisper_ctx_init_openvino_encoder( + self.ctx, + model_path.map(|s| s.as_ptr()).unwrap_or(std::ptr::null()), + device.as_ptr(), + cache_dir.map(|s| s.as_ptr()).unwrap_or(std::ptr::null()), + ) + }; + ret != 0 + } + /// Create a new state object, ready for use. /// /// # Returns diff --git a/sys/Cargo.toml b/sys/Cargo.toml index 9f0db7e..0a284ca 100644 --- a/sys/Cargo.toml +++ b/sys/Cargo.toml @@ -33,6 +33,7 @@ include = [ coreml = [] cuda = [] opencl = [] +openvino = [] [dependencies] diff --git a/sys/build.rs b/sys/build.rs index 2b6b1a1..0f9b596 100644 --- a/sys/build.rs +++ b/sys/build.rs @@ -7,6 +7,26 @@ use std::env; use std::path::PathBuf; fn main() { + // Fail-fast test for OpenVINO + #[cfg(feature = "openvino")] + if let Ok(openvino_dir) = env::var("OpenVINO_DIR") { + // see if we can find OpenVINOConfig.cmake + let openvino_config_path = PathBuf::from(&openvino_dir).join("OpenVINOConfig.cmake"); + if !openvino_config_path.exists() { + panic!( + "Couldn't find OpenVINOConfig.cmake in OpenVINO_DIR. Please set it to the path where `OpenVINOConfig.cmake` can be found.\n\ + On Arch Linux, if you installed the AUR package, this path is `/opt/intel/openvino/runtime/cmake/`.\n\ + Note the `/cmake/` at the end of the path." + ); + } + } else { + panic!( + "Couldn't find the OpenVINO_DIR environment variable. Please set it to the path where `OpenVINOConfig.cmake` can be found.\n\ + On Arch Linux, if you installed the AUR package, this path is `/opt/intel/openvino/runtime/cmake/`." + ); + } + + let target = env::var("TARGET").unwrap(); // Link C++ standard library if let Some(cpp_stdlib) = get_cpp_link_stdlib(&target) { @@ -112,6 +132,9 @@ fn main() { #[cfg(feature = "opencl")] cmd.arg("-DWHISPER_CLBLAST=ON"); + #[cfg(feature = "openvino")] + cmd.arg("-DWHISPER_OPENVINO=1"); + cmd.arg("-DCMAKE_POSITION_INDEPENDENT_CODE=ON"); let code = cmd