stem words for destuttering

hotwords/transcript.sh

@@ -13,6 +13,7 @@ if ! which rust-whisper-baked; then
 fi >&2
 
 cat <<EOF
+rust-whisper-baked --stream-device pulse_monitor --stream-step 16 --stream-retain 8 --stream-{head,tail}=0.25 2> /dev/null
 rust-whisper-baked --stream-device 'BlackHole 2ch' --stream-step 30 --stream-retain 1 --stream-{head,tail}=0.25 --threads 9 2> /dev/null
    | tee -a "$HOME/Sync/drawful/DnD/bdoob/__log.d/$(date +%Y.%m.%d).transcript.txt"
    | tee -a "$HOME/Sync/drawful/DnD/nessira.d/_log.d/$(date +%Y.%m.%d).transcript.txt"

listen-lib/src/lib.rs

@@ -25,7 +25,11 @@ pub fn devices() -> Vec<String> {
 fn _devices() -> Result<Vec<cpal::Device>, String> {
    match cpal::default_host().devices() {
       Ok(devices) => Ok(devices.filter(|device| {
-         device.supported_input_configs().unwrap().count() > 0
+         let input_configs = device.supported_input_configs();
+         if !input_configs.is_ok() {
+            return false;
+         }
+         input_configs.unwrap().count() > 0
       }).collect()),
       Err(msg) => Err(format!("failed to get devices: {}", msg)),
    }
@@ -92,13 +96,22 @@ impl Listener {
          filter(|device| device.name().unwrap() == self.device_name).
          collect::<Vec<_>>();
       let device = devices.first().unwrap();
-      let cfg = device.supported_input_configs()
+      let mut sample_rate = 15_500;
+      let mut cfgs: Vec<_> = device.supported_input_configs()
          .unwrap()
          .filter(|x| x.sample_format() == cpal::SampleFormat::F32)
-         .filter(|x| x.min_sample_rate() >= cpal::SampleRate(15_500))
+         .filter(|x| x.min_sample_rate() >= cpal::SampleRate(sample_rate))
-         .nth(0)
+         .collect();
-         .unwrap()
+      while cfgs.len() == 0 && sample_rate > 0 {
-         .with_max_sample_rate();
+         sample_rate /= 2;
+         cfgs = device.supported_input_configs()
+            .unwrap()
+            .filter(|x| x.sample_format() == cpal::SampleFormat::F32)
+            .filter(|x| x.min_sample_rate() >= cpal::SampleRate(sample_rate))
+            .collect();
+      }
+      assert!(cfgs.len() > 0);
+      let cfg = cfgs[0].clone().with_max_sample_rate();
 
       let downsample_ratio = cfg.channels() as f32 * (cfg.sample_rate().0 as f32 / 16_000.0);
       let stream = device.build_input_stream(

rust-whisper-baked-lib/src/lib.rs

@@ -6,7 +6,7 @@ pub fn channel<F>(
    stream: std::sync::mpsc::Receiver<Vec<f32>>,
 ) where F: FnMut(Result<rust_whisper_lib::Transcribed, String>) + Send + 'static {
    flags.model_path = None;
-   flags.model_buffer = Some(include_bytes!("../../models/ggml-tiny.en.bin").to_vec());
+   flags.model_buffer = Some(get_fast());
    rust_whisper_lib::channel(flags.clone(), handler_fn, stream);
 }
 
@@ -15,7 +15,7 @@ pub fn wav<F>(
    handler_fn: F
 ) where F: FnMut(Result<rust_whisper_lib::Transcribed, String>) + Send + 'static {
    flags.model_path = None;
-   flags.model_buffer = Some(include_bytes!("../../models/ggml-distil-medium.en.bin").to_vec());
+   flags.model_buffer = Some(get_good());
    rust_whisper_lib::wav(flags.clone(), handler_fn, flags.wav.unwrap());
 }
 
@@ -24,10 +24,18 @@ pub fn wav_channel<F>(
    handler_fn: F
 ) where F: FnMut(Result<rust_whisper_lib::Transcribed, String>) + Send + 'static {
    flags.model_path = None;
-   flags.model_buffer = Some(include_bytes!("../../models/ggml-base.en.bin").to_vec());
+   flags.model_buffer = Some(get_good());
    rust_whisper_lib::wav_channel(flags, handler_fn);
 }
 
 pub fn f32_from_wav_file(path: &String) -> Result<Vec<f32>, String> {
    rust_whisper_lib::f32_from_wav_file(path)
 }
+
+fn get_fast() -> Vec<u8> {
+   include_bytes!("../../models/ggml-small.en.bin").to_vec()
+}
+
+fn get_good() -> Vec<u8> {
+   include_bytes!("../../models/ggml-distil-medium.en.bin").to_vec()
+}

rust-whisper-baked/Cargo.lock

@@ -383,6 +383,12 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "itoa"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
+
 [[package]]
 name = "jni"
 version = "0.19.0"
@@ -757,14 +763,26 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"
 
+[[package]]
+name = "rust-stemmers"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
 [[package]]
 name = "rust-whisper-baked"
 version = "0.1.0"
 dependencies = [
  "clap",
  "listen-lib",
+ "rust-stemmers",
  "rust-whisper-baked-lib",
  "rust-whisper-lib",
+ "stop-words",
 ]
 
 [[package]]
@@ -802,6 +820,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -817,6 +841,37 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "serde"
+version = "1.0.193"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.193"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.41",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "shlex"
 version = "1.2.0"
@@ -848,6 +903,15 @@ version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
 
+[[package]]
+name = "stop-words"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8500024d809de02ecbf998472b7bed3c4fca380df2be68917f6a473bdb28ddcc"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "strsim"
 version = "0.10.0"

rust-whisper-baked/Cargo.toml

@@ -10,3 +10,5 @@ rust-whisper-lib = { path = "../rust-whisper-lib" }
 rust-whisper-baked-lib = { path = "../rust-whisper-baked-lib" }
 listen-lib = { path = "../listen-lib" }
 clap = { version = "4.4.10", features = ["derive"] }
+stop-words = "0.8.0"
+rust-stemmers = "1.2.0"

rust-whisper-baked/src/main.rs

@@ -48,11 +48,15 @@ fn channel(flags: rust_whisper_lib::Flags) {
 
    eprintln!("rust whisper baked lib channel...");
    thread::spawn(move || {
+      let mut w = new_destutterer();
       rust_whisper_baked_lib::channel(
          flags.clone(),
-         |result: Result<rust_whisper_lib::Transcribed, String>| {
+         move |result: Result<rust_whisper_lib::Transcribed, String>| {
             match result {
-               Ok(transcribed) => { println!("{}", transcribed.to_string()); },
+               Ok(transcribed) => {
+                  let s = w.step(transcribed.to_string());
+                  println!("{}", s);
+               },
                Err(msg) => { eprintln!("error: {}", msg); },
             };
          },
@@ -64,10 +68,14 @@ fn channel(flags: rust_whisper_lib::Flags) {
    let flags = rust_whisper_lib::Flags::parse();
    match flags.stream_device {
       Some(device_name) => {
-         if device_name == "" {
+         eprintln!("with device ({}) '{}'", device_name.len(), &device_name);
+         if device_name.len() == 0 {
+            let mut i = 0;
             for device in listen_lib::devices() {
-               eprintln!("{}", device);
+               eprintln!("[{}] {}", i, device);
+               i += 1;
             }
+            eprintln!("found {} devices", i);
          } else {
             listen_lib::main_with(|data| {
                send.send(data).unwrap();
@@ -75,6 +83,7 @@ fn channel(flags: rust_whisper_lib::Flags) {
          }
       },
       None => {
+         eprintln!("without any device");
          listen_lib::main(|data| {
             send.send(data).unwrap();
          });
@@ -84,52 +93,119 @@ fn channel(flags: rust_whisper_lib::Flags) {
 }
 
 struct Destutterer {
-   prev: Option<String>,
+   prev: Words,
 }
 
 fn new_destutterer() -> Destutterer {
-   Destutterer{prev: None}
+   Destutterer{prev: new_words()}
 }
 
 impl Destutterer {
    fn step(&mut self, next: String) -> String {
-      let next = next.trim().to_string();
       if next.len() == 0 {
          return next;
       }
-      match &self.prev {
+
-         None => {
+      let next_words = Words::from_string(next.clone());
-            self.prev = Some(next.clone());
+      let mut n = self.prev.comparable_len().clamp(0, next_words.comparable_len());
-            next
+      //println!("n={} prev='{:?}' next='{:?}'", n, self.prev.to_comparable_words(), next_words.to_comparable_words());
-         },
+      while n > 0 {
-         Some(prev) => {
+         let (prev_s, _) = self.prev.last_n_comparable_to_string(n);
-            let without_trailing_punctuation = {
+         let (next_s, next_idx) = next_words.first_n_comparable_to_string(n);
-               let mut next = next.clone();
+         if prev_s == next_s {
-               while next.ends_with("?") || next.ends_with(".") {
+            self.prev = next_words;
-                  next = next[..next.len()-1].to_string();
+            return self.prev.skip(next_idx+1).to_string();
-               }
+         }
-               next
+         n -= 1;
-            };
+      }
-            let trailing_punctuation = next[without_trailing_punctuation.len() ..].to_string();
+      self.prev = next_words;
-            let next = without_trailing_punctuation;
+      self.prev.to_string()
-            let next = {
+   }
-               let mut n = prev.len().clamp(0, next.len());
+}
-               while n > 0 {
+
-                  if prev[prev.len() - n..] == next[..n] {
+#[derive(Clone, Debug)]
-                     break;
+struct Words {
-                  }
+   raw: Vec<String>,
-                  n -= 1;
+}
-               }
+
-               next[n..].to_string()
+fn new_words() -> Words {
-            };
+   Words{raw: vec![]}
-            if next.len() == 0 {
+}
-               return "".to_string();
+
-            }
+impl Words {
-            self.prev = Some(next.clone());
+   fn from_string(s: String) -> Words {
-            next + &trailing_punctuation
+      let mut result = Words{raw: vec![]};
-         },
+      for word in s.split(" ") {
+         let word = word.trim();
+         if word.len() > 0 {
+            result.raw.push(word.to_string());
+         }
+      }
+      result
+   }
+
+   fn skip(&self, n: usize) -> Words {
+      Words{
+         raw: self.raw.iter().skip(n).map(|x| x.clone()).collect(),
       }
    }
+
+   fn last_n_comparable_to_string(&self, n: usize) -> (String, usize) {
+      let v = self.to_comparable_words();
+      let v = v[(v.len() - n).clamp(0, v.len())..].to_vec();
+      return (v.iter().map(|x| x.s.clone().unwrap()).collect::<Vec<String>>().join(" "), v[0].idx)
+   }
+
+   fn first_n_comparable_to_string(&self, n: usize) -> (String, usize){
+      let v = self.to_comparable_words();
+      let v = v[0..n.clamp(0, v.len())].to_vec();
+      return (v.iter().map(|x| x.s.clone().unwrap()).collect::<Vec<String>>().join(" "), v[v.len()-1].idx)
+   }
+
+   fn comparable_len(&self) -> usize {
+      self.to_comparable_words().len()
+   }
+
+   fn to_comparable_words(&self) -> Vec<Word> {
+      self.to_words().iter()
+         .filter(|x| x.s.is_some())
+         .map(|x| x.clone())
+         .collect()
+   }
+
+   fn to_words(&self) -> Vec<Word> {
+      let skips = stop_words::get("en");
+      let stemmer = rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::English);
+      let strs = self.raw.iter()
+         .map(|w| w.to_lowercase())
+         .map(|w| w.chars().filter(|c| c.is_ascii_alphanumeric()).collect::<String>())
+         .map(|w| stemmer.stem(&w).into_owned())
+         .collect::<Vec<String>>();
+      let mut result = vec![];
+      for i in 0..strs.len() {
+         result.push(Word{
+            s: match skips.contains(&strs[i]) {
+               true => None,
+               false => Some(strs[i].clone()),
+            },
+            idx: i as usize,
+         });
+      }
+      result
+   }
+
+   fn to_string(&self) -> String {
+      self.raw.iter()
+         .map(|x| x.clone())
+         .collect::<Vec<String>>()
+         .join(" ")
+   }
+}
+
+#[derive(Debug, Clone)]
+struct Word {
+   s: Option<String>,
+   idx: usize,
 }
 
 #[cfg(test)]
@@ -137,11 +213,24 @@ mod tests {
    use super::*;
 
    #[test]
-   fn test_destutterer() {
+   fn test_destutterer_stop_words() {
       let mut w = new_destutterer();
-      assert_eq!("abcde".to_string(), w.step("abcde".to_string()));
+      assert_eq!("welcome to the internet".to_string(), w.step("welcome to the internet".to_string()));
-      assert_eq!("fg".to_string(), w.step("cdefg".to_string()));
+      assert_eq!("have a look around".to_string(), w.step("welcome to the a internet; have a look around".to_string()));
-      assert_eq!("hij".to_string(), w.step("fghij".to_string()));
+   }
-      assert_eq!("fghij".to_string(), w.step("fghij".to_string()));
+
+   #[test]
+   fn test_destutterer_punctuation() {
+      let mut w = new_destutterer();
+      assert_eq!("cat, dog. cow? moose!".to_string(), w.step("cat, dog. cow? moose!".to_string()));
+      assert_eq!("elephant! fez gator".to_string(), w.step("moose, elephant! fez gator".to_string()));
+      assert_eq!("hij".to_string(), w.step("fez gator hij".to_string()));
+   }
+
+   #[test]
+   fn test_destutterer_basic() {
+      let mut w = new_destutterer();
+      assert_eq!("cat dog cow".to_string(), w.step(" cat dog cow ".to_string()));
+      assert_eq!("moose".to_string(), w.step(" dog cow moose  ".to_string()));
    }
 }

rust-whisper-lib/src/lib.rs

@@ -14,13 +14,13 @@ pub struct Flags {
    #[arg(long, default_value = "8")]
    pub threads: i32,
 
-   #[arg(long, default_value = "5")]
+   #[arg(long, default_value = "8")]
    pub stream_step: u64,
-   #[arg(long, default_value = "0.6")]
+   #[arg(long, default_value = "4.0")]
    pub stream_retain: f32,
-   #[arg(long, default_value = "0.3")]
+   #[arg(long, default_value = "2.0")]
    pub stream_head: f32,
-   #[arg(long, default_value = "0.3")]
+   #[arg(long, default_value = "0.0")]
    pub stream_tail: f32,
 
    #[arg(long, default_value = "false")]

todo.yaml

@@ -0,0 +1,11 @@
+todo:
+- compound words like checkmark vs check mark should destutter
+- whisper trims outside silence so head and tail never get hit
+- split on silence-ish instead of duration
+- rust-whisper warn when transcription time ~ input time
+scheduled: []
+done:
+- todo: need to overlap without ANY puctuation, which i can do by breaking into words
+  ts: Tue Jan  2 18:23:00 MST 2024
+- todo: overlap without stop words
+  ts: Wed Jan  3 08:22:14 MST 2024