diff --git a/sys/Cargo.toml b/sys/Cargo.toml
index b429860..1966542 100644
--- a/sys/Cargo.toml
+++ b/sys/Cargo.toml
@@ -46,6 +46,7 @@ cuda = []
 opencl = []
 openblas = []
 metal = []
+force-debug = []
 
 [build-dependencies]
 cmake = "0.1"
diff --git a/sys/build.rs b/sys/build.rs
index 9545b61..1628bf5 100644
--- a/sys/build.rs
+++ b/sys/build.rs
@@ -138,6 +138,12 @@ fn main() {
         config.define("WHISPER_METAL", "OFF");
     }
 
+    if cfg!(debug_assertions) || cfg!(feature = "force-debug") {
+        // debug builds are too slow to even remotely be usable,
+        // so we build with optimizations even in debug mode
+        config.define("CMAKE_BUILD_TYPE", "RelWithDebInfo");
+    }
+
     let destination = config.build();
 
     if env::var("TARGET").unwrap().contains("window") {