From 1a9f05239653cf896546e0b04e4e2fa0566ce090 Mon Sep 17 00:00:00 2001 From: Bel LaPointe <153096461+breel-render@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:01:35 -0700 Subject: [PATCH] new tests with just add-remove and persist_stage collapses finding missed persists --- pttodoest/src/main.rs | 220 ++++++++++++++++++++++++++---------------- 1 file changed, 137 insertions(+), 83 deletions(-) diff --git a/pttodoest/src/main.rs b/pttodoest/src/main.rs index e8fc34b..698d112 100755 --- a/pttodoest/src/main.rs +++ b/pttodoest/src/main.rs @@ -9,21 +9,15 @@ fn main() { if !flags.dry_run { for file in files.files.iter() { - file.stage_new_persisted() - .expect("failed to stage new log files"); file.persist_stage() .expect("failed to persist staged changes to log file"); file.stage_persisted().expect("failed to stage log files"); } if let Some(add) = flags.add { - let patch: json_patch::PatchOperation = - json_patch::PatchOperation::Add(json_patch::AddOperation { - path: jsonptr::PointerBuf::parse("/-").expect("cannot create path to /-"), - value: serde_json::json!(add), - }); + let task = Task(serde_yaml::Value::String(add)); files.files[0] - .append(Delta::now(patch)) + .append(Delta::add(task)) .expect("failed to add"); if !flags.enqueue_add { files.files[0] @@ -159,7 +153,7 @@ impl File { Events::new(&self.file) } - pub fn stage_new_persisted(&self) -> Result<(), String> { + pub fn persist_unpersisted_stage(&self) -> Result<(), String> { let events = self.events()?; let stage_mod_time = std::fs::metadata(&self.file) .unwrap() @@ -168,14 +162,16 @@ impl File { .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(); - let new_persisted: Vec = events + let old_persisted: Vec = events .0 .iter() - .filter(|x| x.ts > stage_mod_time) + .filter(|x| x.ts < stage_mod_time) .map(|x| x.clone()) .collect(); - panic!("not impl: apply filtered deltas to stage"); - Ok(()) + let old_events = Events(old_persisted); + let old_snapshot = old_events.snapshot()?; + self.persist_delta_at(old_snapshot, self.stage()?, stage_mod_time)?; + self.stage_persisted() } pub fn stage_persisted(&self) -> Result<(), String> { @@ -187,6 +183,8 @@ impl File { } pub fn persist_stage(&self) -> Result<(), String> { + self.persist_unpersisted_stage()?; + let persisted = self.events()?.snapshot()?; let stage = self.stage()?; @@ -195,21 +193,24 @@ impl File { } pub fn persist_delta(&self, before: Vec, after: Vec) -> Result<(), String> { - let before = serde_json::to_string(&before).unwrap(); - let before = before.as_str(); - let before: serde_json::Value = serde_json::from_str(&before).unwrap(); + self.persist_delta_at(before, after, Delta::now_time()) + } - let after = serde_json::to_string(&after).unwrap(); - let after: serde_json::Value = serde_json::from_str(after.as_str()).unwrap(); - - let patches = json_patch::diff(&before, &after); - let deltas: Vec = patches - .iter() - .map(|patch| patch.clone()) - .map(|patch| Delta::now(patch.clone())) - .collect(); - for delta in deltas.iter() { - self.append(delta.clone())?; + fn persist_delta_at( + &self, + before: Vec, + after: Vec, + now: u64, + ) -> Result<(), String> { + for before in before.iter() { + if !after.contains(before) { + self.append(Delta::remove_at(before.clone(), now)); + } + } + for after in after.iter() { + if !before.contains(after) { + self.append(Delta::add_at(after.clone(), now)); + } } Ok(()) } @@ -305,7 +306,8 @@ mod test_file { f.persist_stage().unwrap(); assert_eq!(2, f.events().unwrap().0.len()); assert_eq!(2, f.stage().unwrap().len()); - tests::file_contains(&d, "plain", "[hello, world]"); + tests::file_contains(&d, "plain", "hello"); + tests::file_contains(&d, "plain", "world"); f.stage_persisted().unwrap(); assert_eq!(2, f.events().unwrap().0.len()); @@ -322,15 +324,15 @@ mod test_file { &d, ".plain.host_a", r#" - {"ts":1, "patch":{"op":"replace", "path":"", "value": ["initial"]}} - {"ts":3, "patch":{"op":"add", "path":"/-", "value": {"k":"v"}}} + {"ts":1, "op":"Add", "task": "initial"} + {"ts":3, "op":"Add", "task": {"k":"v"}} "#, ); tests::write_file( &d, ".plain.host_b", r#" - {"ts":2, "patch":{"op":"add", "path":"/-", "value": 1}} + {"ts":2, "op":"Add", "task": 1} "#, ); @@ -343,11 +345,28 @@ mod test_file { f.persist_stage().unwrap(); assert_eq!(6, f.events().unwrap().0.len()); assert_eq!(0, f.stage().unwrap().len()); + eprintln!("persist_stage | events | {:?}", f.events().unwrap().0); + eprintln!( + "persist_stage | events.snapshot | {:?}", + f.events().unwrap().snapshot() + ); + eprintln!("persist_stage | stage | {:?}", f.stage().unwrap()); tests::file_contains(&d, "plain", "[]"); f.stage_persisted().unwrap(); - assert_eq!(6, f.events().unwrap().0.len()); - assert_eq!(0, f.stage().unwrap().len()); + assert_eq!( + 0, + f.events().unwrap().snapshot().unwrap().len(), + "{:?}", + f.events().unwrap().snapshot().unwrap(), + ); + assert_eq!( + 6, + f.events().unwrap().0.len(), + "{:?}", + f.events().unwrap().0 + ); + assert_eq!(0, f.stage().unwrap().len(), "{:?}", f.stage().unwrap()); tests::file_contains(&d, "plain", "[]"); }); } @@ -360,15 +379,15 @@ mod test_file { &d, ".plain.host_a", r#" - {"ts":1, "patch":{"op":"replace", "path":"", "value": ["initial"]}} - {"ts":3, "patch":{"op":"add", "path":"/-", "value": {"k":"v"}}} + {"ts":1, "op":"Add", "task": "initial"} + {"ts":3, "op":"Add", "task": {"k":"v"}} "#, ); tests::write_file( &d, ".plain.host_b", r#" - {"ts":2, "patch":{"op":"add", "path":"/-", "value": 1}} + {"ts":2, "op":"Add", "task": 1} "#, ); @@ -391,7 +410,7 @@ mod test_file { } #[test] - fn test_stage_new_persisted() { + fn test_persist_unpersisted_stage() { tests::with_dir(|d| { tests::write_file(&d, "plain", "- old\n- new"); tests::write_file( @@ -399,36 +418,39 @@ mod test_file { ".plain.host", format!( r#" - {{"ts":{}, "patch":{{"op":"replace", "path":"/0", "value": "enqueued for persistence"}}}} + {{"ts":1, "op":"Add", "task": "removed"}} + {{"ts":2, "op":"Add", "task": "old"}} + {{"ts":{}, "op":"Add", "task": "enqueued for persistence"}} "#, 2147483647, - ).as_str(), + ) + .as_str(), ); let f = File::new(&d.path().join("plain").to_str().unwrap().to_string()); - assert_eq!(1, f.events().unwrap().0.len()); + assert_eq!(3, f.events().unwrap().0.len()); assert_eq!(2, f.stage().unwrap().len()); tests::file_contains(&d, "plain", "old"); tests::file_contains(&d, "plain", "new"); - f.stage_new_persisted().unwrap(); - tests::file_contains(&d, "plain", "enqueued"); - tests::file_contains(&d, "plain", "new"); - assert_eq!(1, f.events().unwrap().0.len()); - assert_eq!(2, f.stage().unwrap().len()); - f.persist_stage().unwrap(); - assert_eq!(3, f.events().unwrap().0.len()); - assert_eq!(2, f.stage().unwrap().len()); + assert_eq!(5, f.events().unwrap().0.len()); + assert_eq!(3, f.stage().unwrap().len()); tests::file_contains(&d, "plain", "enqueued"); tests::file_contains(&d, "plain", "new"); f.stage_persisted().unwrap(); - assert_eq!(3, f.events().unwrap().0.len()); - assert_eq!(2, f.stage().unwrap().len()); + assert_eq!( + 5, + f.events().unwrap().0.len(), + "{:?}", + f.events().unwrap().0 + ); + assert_eq!(3, f.stage().unwrap().len(), "{:?}", f.stage().unwrap()); tests::file_contains(&d, "plain", "enqueued"); tests::file_contains(&d, "plain", "new"); + tests::file_contains(&d, "plain", "old"); }); } } @@ -436,27 +458,52 @@ mod test_file { #[derive(Debug, Clone, Serialize, Deserialize)] struct Delta { ts: u64, - patch: json_patch::PatchOperation, + op: Op, + task: Task, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum Op { + Add, + Remove, } impl Delta { - pub fn new(patch: json_patch::PatchOperation, ts: u64) -> Delta { + pub fn new(ts: u64, op: Op, task: Task) -> Delta { Delta { - patch: patch, ts: ts, + op: op, + task: task, } } - pub fn now(patch: json_patch::PatchOperation) -> Delta { - Self::new( - patch, - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs() - .try_into() - .unwrap(), - ) + pub fn add(task: Task) -> Delta { + Self::add_at(task, Self::now_time()) + } + + pub fn add_at(task: Task, at: u64) -> Delta { + Self::new(at, Op::Add, task) + } + + pub fn remove(task: Task) -> Delta { + Self::remove_at(task, Self::now_time()) + } + + pub fn remove_at(task: Task, at: u64) -> Delta { + Self::new(at, Op::Remove, task) + } + + pub fn now(op: Op, task: Task) -> Delta { + Self::new(Self::now_time(), op, task) + } + + fn now_time() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + .try_into() + .unwrap() } } @@ -528,20 +575,26 @@ impl Events { } fn snapshot(&self) -> Result, String> { - let mut result = serde_json::json!([]); + let mut result = vec![]; for event in self.0.iter() { - match json_patch::patch(&mut result, vec![event.patch.clone()].as_slice()) { - Ok(_) => Ok(()), - Err(msg) => Err(format!( - "failed to patch {} onto {}: {}", - &event.patch, &result, msg - )), - }?; - } - match serde_json::from_str(serde_json::to_string(&result).unwrap().as_str()) { - Ok(v) => Ok(v), - Err(msg) => Err(format!("failed turning patched into events: {}", msg)), + match event.op { + Op::Add => result.push(event.task.clone()), + Op::Remove => { + let mut i = (result.len() - 1) as i32; + while i >= 0 { + if event.task == result[i as usize] { + result.remove(i as usize); + if i == result.len() as i32 { + i -= 1 + } + } else { + i -= 1; + } + } + } + }; } + Ok(result) } } @@ -557,7 +610,7 @@ mod test_events { &d, ".plain.some_host", r#" - {"ts":1, "patch":{"op":"replace", "path":"", "value":["persisted"]}} + {"ts":1, "op":"Add", "task":"persisted"} "#, ); @@ -582,19 +635,20 @@ mod test_events { &d, ".plain.host_a", r#" - {"ts":1, "patch":{"op":"replace", "path":"", "value":["persisted"]}} - {"ts":3, "patch":{"op":"add", "path":"/-", "value":"persisted 3"}} - {"ts":2, "patch":{"op":"add", "path":"/-", "value":"persisted 2"}} - {"ts":6, "patch":{"op":"replace", "path":"/4", "value":"persisted 5'"}} - {"ts":7, "patch":{"op":"remove", "path":"/3"}} + {"ts":1, "op":"Add", "task":"persisted"} + {"ts":3, "op":"Add", "task":"persisted 3"} + {"ts":2, "op":"Add", "task":"persisted 2"} + {"ts":6, "op":"Remove", "task":"persisted 5"} + {"ts":6, "op":"Add", "task":"persisted 5'"} + {"ts":7, "op":"Remove", "task":"persisted 4"} "#, ); tests::write_file( &d, ".plain.host_b", r#" - {"ts":4, "patch":{"op":"add", "path":"/-", "value":"persisted 4"}} - {"ts":5, "patch":{"op":"add", "path":"/-", "value":"persisted 5"}} + {"ts":4, "op":"Add", "task":"persisted 4"} + {"ts":5, "op":"Add", "task":"persisted 5"} "#, );