From f3b960c2f57306d8130c5674f83ce89e6fbca9df Mon Sep 17 00:00:00 2001 From: Bel LaPointe <153096461+breel-render@users.noreply.github.com> Date: Wed, 27 Dec 2023 21:42:53 -0500 Subject: [PATCH] from jpg to png --- src-lib/Cargo.lock | 2 +- src-lib/_all | 375 +++++++++++++++++++++++++++++++++++++++++++ src-lib/src/video.rs | 36 ++++- 3 files changed, 405 insertions(+), 8 deletions(-) create mode 100644 src-lib/_all diff --git a/src-lib/Cargo.lock b/src-lib/Cargo.lock index 1d866bc..213cc1f 100644 --- a/src-lib/Cargo.lock +++ b/src-lib/Cargo.lock @@ -3,5 +3,5 @@ version = 3 [[package]] -name = "src-lib" +name = "lib" version = "0.1.0" diff --git a/src-lib/_all b/src-lib/_all new file mode 100644 index 0000000..f06a36e --- /dev/null +++ b/src-lib/_all @@ -0,0 +1,375 @@ +use std::str::FromStr; +use core::cmp::Ordering; +use std::fs; +use std::path::Path; +use std::io::Write; + +pub fn screenshotify(input: &String) -> Result, String> { + let output_d = format!("{}.d", input); + let ext = "jpg".to_string(); + let mut result = vec![]; + let _ = ify(input, |content_span| { + let output = format!("{}/{}.{}", output_d, result.len(), ext); + let _ = screenshot(&output, input, (content_span.stop + content_span.start) / 2.0)?; + result.push(output); + Ok(()) + })?; + Ok(result) +} + +pub fn clipify(input: &String) -> Result, String> { + let output_d = format!("{}.d", input); + let ext = input.split(".").last().unwrap(); + let mut result = vec![]; + let _ = ify(input, |content_span| { + let output = format!("{}/{}.{}", output_d, result.len(), ext); + let _ = clip(&output, input, content_span)?; + result.push(output); + Ok(()) + })?; + Ok(result) +} + +fn ify(input: &String, mut cb: impl FnMut(ContentSpan) -> Result<(), String>) -> Result<(), String> { + match inspect(input) { + Ok(inspection) => { + match inspection.iter() + .map(|x| cb(*x)) + .filter(|x| x.is_err()) + .map(|x| x.err()) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .nth(0) { + Some(err) => Err(format!("callback failed: {}", err)), + None => Ok(()), + } + }, + Err(msg) => Err(msg), + } +} + +pub fn clip(output: &String, input: &String, content_span: ContentSpan) -> Result<(), String> { + fs::create_dir_all(Path::new(output).parent().unwrap()).unwrap(); + match std::process::Command::new("ffmpeg") + .args([ + "-y", + "-ss", &content_span.start.to_string(), + "-i", input, + "-t", &(content_span.stop - content_span.start).to_string(), + output, + ]) + .output() { + Ok(output) => match output.status.success() { + true => Ok(()), + false => Err(format!("failed to ffmpeg clip {}: {}", input, String::from_utf8(output.stderr).unwrap())), + }, + Err(msg) => Err(format!("failed to ffmpeg clip {}: {}", input, msg)), + } +} + +pub fn screenshot(output: &String, input: &String, ts: f32) -> Result<(), String> { + match screenshot_jpg(input, ts) { + Ok(jpg) => { + match std::fs::File::create(output) { + Ok(mut f) => { + f.write_all(jpg).unwrap(); + Ok(()) + }, + Err(msg) => Err(format!("failed to open {} for writing: {}", output, msg)), + } + }, + Err(msg) => Err(msg), + } +} + +pub fn screenshot_jpg(input: &String, ts: f32) -> Result, String> { + match std::process::Command::new("ffmpeg") + .args([ + "-y", + "-ss", &ts.to_string(), + "-i", input, + "-frames:v", "1", + "-q:v", "2", + "-f", "jpg", + "-", + ]) + .output() { + Ok(output) => Ok(output.stdout), + Err(msg) => Err(format!("failed to ffmpeg screenshot {}: {}", input, msg)), + } +} + +pub fn inspect(file: &String) -> Result, String> { + match _inspect(file) { + Ok(inspection) => Ok(inspection.content_spans()), + Err(msg) => Err(msg) + } +} + +fn _inspect(file: &String) -> Result { + match std::process::Command::new("ffmpeg") + .args([ + "-i", file, + "-vf", "mpdecimate", + "-af", "silencedetect=n=-50dB:d=1", + "-loglevel", "debug", + "-f", "null", + "-", + ]) + .output() { + Ok(output) => { + let stderr = String::from_utf8(output.stderr).unwrap(); + let stdout = String::from_utf8(output.stdout).unwrap(); + let line_iter = stderr.split("\n").chain(stdout.split("\n")); + Ok(Inspection{ + lines: line_iter.map(|x| x.to_string()).collect(), + }) + }, + Err(msg) => Err(format!("failed to ffmpeg inspect {}: {}", file, msg)), + } +} + +struct Inspection { + lines: Vec, +} + +impl Inspection { + fn content_spans(&self) -> Vec { + self.overlap_spans( + &self.visual_spans(), + &self.audible_spans(), + ) + } + + fn overlap_spans(&self, a: &Vec, b: &Vec) -> Vec { + self.merge_unsorted_spans(a.iter().chain(b.iter()).map(|x| *x).collect()) + } + + fn audible_spans(&self) -> Vec { + self.spans_from_transitions(self.audible_transitions()) + } + + fn visual_spans(&self) -> Vec { + self.spans_from_transitions(self.visual_transitions()) + } + + fn visual_transitions(&self) -> Vec { + let lines: Vec<_> = self.lines.iter() + .filter(|x| x.contains("Parsed_mpdecimate_0")) + .filter(|x| x.contains("keep pts:") || x.contains("drop pts:")) + .collect(); + let mut lines_with_transitions = vec![]; + for i in 0..lines.len() { + if lines_with_transitions.len() == 0 { + lines_with_transitions.push(lines[i]); + } else { + let this_is_keep = lines[i].contains("keep"); + let prev_was_keep = lines_with_transitions.last().unwrap().contains("keep"); + if this_is_keep != prev_was_keep { + lines_with_transitions.push(lines[i-1]); + lines_with_transitions.push(lines[i]); + } + } + } + lines_with_transitions.iter() + .skip(2) + .filter(|x| x.contains("keep")) + .map(|x| x.split("pts_time:").nth(1).unwrap().split(" ").nth(0).unwrap()) + .map(|x| f32::from_str(x).unwrap()) + .collect() + } + + fn audible_transitions(&self) -> Vec { + let lines: Vec<_> = self.lines.iter() + .filter(|x| x.contains("[silencedetect @")) + .map(|x| x.split(" ").nth(4).unwrap()) + .map(|x| f32::from_str(x).unwrap()) + .collect(); + let mut lines_with_transitions = vec![]; + lines_with_transitions.push(0.0); + lines.iter().for_each(|x| lines_with_transitions.push(*x)); + lines_with_transitions.push(self.duration()); + lines_with_transitions + } + + fn duration(&self) -> f32 { + let ts = self.lines.iter() + .filter(|x| (*x).contains("Duration: ")) + .filter(|x| (*x).contains("start: ")) + .filter(|x| (*x).contains("bitrate: ")) + .nth(0) + .expect("did not find duration from ffmpeg") + .split(",").nth(0).unwrap() + .split(": ").nth(1).unwrap(); + let pieces: Vec<_> = ts.split(":") + .map(|x| f32::from_str(x).unwrap()) + .collect(); + assert_eq!(3, pieces.len()); + let hours = pieces[0] * 60.0 * 60.0; + let minutes = pieces[1] * 60.0; + let seconds = pieces[2]; + hours + minutes + seconds + } + + fn spans_from_transitions(&self, transitions: Vec) -> Vec { + let spans: Vec<_> = self.raw_spans_from_transitions(transitions).iter() + .filter(|x| x.stop - x.start > 0.25) // TODO const + .map(|x| ContentSpan{ + start: (x.start - 1.0).clamp(0.0, self.duration()), + stop: (x.stop + 1.0).clamp(0.0, self.duration()), // TODO const + }) + .collect(); + self.merge_spans(spans) + } + + fn merge_unsorted_spans(&self, spans: Vec) -> Vec { + let mut spans: Vec<_> = spans.iter().map(|x| *x).collect(); + spans.sort_by(|x, y| match x.start.partial_cmp(&y.start).unwrap() { + Ordering::Less => Ordering::Less, + Ordering::Equal => x.stop.partial_cmp(&y.stop).unwrap(), + Ordering::Greater => Ordering::Greater, + }); + self.merge_spans(spans) + } + + fn merge_spans(&self, spans: Vec) -> Vec { + let mut result: Vec<_> = vec![]; + let mut cur = *spans.iter().nth(0).or(Some(&ContentSpan{start: 0.0, stop: 0.0})).unwrap(); + for i in 1..spans.len() { + if spans[i].start - cur.stop > 5.0 { // TODO const + if !cur.empty() { + result.push(cur); + } + cur = spans[i]; + } else if spans[i].stop > cur.stop { + cur.stop = spans[i].stop + } + } + result.push(cur); + result + } + + fn raw_spans_from_transitions(&self, transitions: Vec) -> Vec { + let mut result = vec![]; + for i in (0..transitions.len()).step_by(2) { + let start = transitions[i]; + let mut stop = self.duration(); + if transitions.len() > i+1 { + stop = transitions[i+1]; + } + result.push(ContentSpan{start: start, stop: stop}); + } + result + } +} + +#[cfg(test)] +mod test_inspection { + use super::*; + const FILE: &str = "/Users/breel/Movies/bel_1_1.mp4"; + + #[test] + fn test_screenshotify() { + let result = screenshotify(&FILE.to_string()).unwrap(); + for i in result.iter() { + eprintln!("{}", i); + } + } + + #[test] + fn test_clipify() { + let result = clipify(&FILE.to_string()).unwrap(); + for i in result.iter() { + eprintln!("{}", i); + } + } + + #[test] + fn test_screenshot() { + let output = format!("{}.clipped.jpg", FILE); + screenshot( + &output, + &FILE.to_string(), + 3.0, + ).unwrap(); + + let inspection = _inspect(&output); + assert_eq!(true, inspection.is_ok()); + let inspection = inspection.unwrap(); + assert_eq!(0.04, inspection.duration()); + } + + #[test] + fn test_clip() { + let output = format!("{}.clipped.mp4", FILE); + clip( + &output, + &FILE.to_string(), + ContentSpan{start: 3.0, stop: 5.0}, + ).unwrap(); + + let inspection = _inspect(&output); + assert_eq!(true, inspection.is_ok()); + let inspection = inspection.unwrap(); + assert_eq!(2.0, inspection.duration()); + } + + #[test] + fn test_inspect() { + let inspection = _inspect(&FILE.to_string()); + assert_eq!(true, inspection.is_ok()); + let inspection = inspection.unwrap(); + + assert_eq!(28.14, inspection.duration()); + + assert_eq!(4, inspection.visual_transitions().len()); + assert_eq!( + vec![ContentSpan{start: 1.0520501, stop: 22.0043}], + inspection.visual_spans(), + ); + + assert_eq!(8, inspection.audible_transitions().len()); + assert_eq!( + vec![ContentSpan{start: 3.3723102, stop: 20.8207}], + inspection.audible_spans(), + ); + + assert_eq!(1, inspection.content_spans().len()); + assert_eq!( + vec![ContentSpan{start: 1.0520501, stop: 22.0043}], + inspection.content_spans(), + ); + + let lines = inspection.lines; + assert!(lines.iter().filter(|x| x.contains("Parsed_mpdecimate_0")).count() > 0); + assert!(lines.iter().filter(|x| x.contains("Duration: 00")).count() > 0); + assert!(lines.iter().filter(|x| x.contains("silence_end: ")).count() > 0); + assert!(lines.iter().filter(|x| x.contains("silence_start: ")).count() > 0); + } +} + +#[derive(Debug, PartialEq, Clone, Copy, PartialOrd)] +pub struct ContentSpan { + pub start: f32, + pub stop: f32, +} + +impl ContentSpan { + pub fn duration(&self) -> f32 { + self.stop - self.start + } + + fn empty(&self) -> bool { + *self == (ContentSpan{start: 0.0, stop: 0.0}) + } +} + +#[cfg(test)] +mod test_content_span { + use super::*; + + #[test] + fn test_duration() { + assert_eq!(1.0, ContentSpan{start: 1.0, stop: 2.0}.duration()); + } +} diff --git a/src-lib/src/video.rs b/src-lib/src/video.rs index 25b4530..c0814eb 100644 --- a/src-lib/src/video.rs +++ b/src-lib/src/video.rs @@ -2,10 +2,11 @@ use std::str::FromStr; use core::cmp::Ordering; use std::fs; use std::path::Path; +use std::io::Write; pub fn screenshotify(input: &String) -> Result, String> { let output_d = format!("{}.d", input); - let ext = "jpg".to_string(); + let ext = "png".to_string(); let mut result = vec![]; let _ = ify(input, |content_span| { let output = format!("{}/{}.{}", output_d, result.len(), ext); @@ -67,6 +68,21 @@ pub fn clip(output: &String, input: &String, content_span: ContentSpan) -> Resul } pub fn screenshot(output: &String, input: &String, ts: f32) -> Result<(), String> { + match screenshot_png(input, ts) { + Ok(png) => { + match std::fs::File::create(output) { + Ok(mut f) => { + f.write_all(&png).unwrap(); + Ok(()) + }, + Err(msg) => Err(format!("failed to open {} for writing: {}", output, msg)), + } + }, + Err(msg) => Err(msg), + } +} + +pub fn screenshot_png(input: &String, ts: f32) -> Result, String> { match std::process::Command::new("ffmpeg") .args([ "-y", @@ -74,10 +90,12 @@ pub fn screenshot(output: &String, input: &String, ts: f32) -> Result<(), String "-i", input, "-frames:v", "1", "-q:v", "2", - output, + "-c:v", "png", + "-f", "image2pipe", + "-", ]) .output() { - Ok(_) => Ok(()), + Ok(output) => Ok(output.stdout), Err(msg) => Err(format!("failed to ffmpeg screenshot {}: {}", input, msg)), } } @@ -178,15 +196,19 @@ impl Inspection { fn duration(&self) -> f32 { let ts = self.lines.iter() .filter(|x| (*x).contains("Duration: ")) - .filter(|x| (*x).contains("start: ")) .filter(|x| (*x).contains("bitrate: ")) .nth(0) .expect("did not find duration from ffmpeg") .split(",").nth(0).unwrap() .split(": ").nth(1).unwrap(); let pieces: Vec<_> = ts.split(":") - .map(|x| f32::from_str(x).unwrap()) + .map(|x| f32::from_str(x)) + .filter(|x| x.is_ok()) + .map(|x| x.unwrap()) .collect(); + if pieces.len() == 0 { + return 0.0; + } assert_eq!(3, pieces.len()); let hours = pieces[0] * 60.0 * 60.0; let minutes = pieces[1] * 60.0; @@ -269,7 +291,7 @@ mod test_inspection { #[test] fn test_screenshot() { - let output = format!("{}.clipped.jpg", FILE); + let output = format!("{}.clipped.png", FILE); screenshot( &output, &FILE.to_string(), @@ -279,7 +301,7 @@ mod test_inspection { let inspection = _inspect(&output); assert_eq!(true, inspection.is_ok()); let inspection = inspection.unwrap(); - assert_eq!(0.04, inspection.duration()); + assert_eq!(0.0, inspection.duration()); } #[test]