testdata to new NOT jsonpatch

pttodoest/src/main.rs

@@ -153,21 +153,8 @@ impl File {
         Events::new(&self.file)
     }
 
-    fn events_non_future(&self) -> Result<Events, String> {
-        let events = self.events()?;
-        let now = Delta::now_time();
-        Ok(Events(
-            events
-                .0
-                .iter()
-                .filter(|x| x.ts <= now)
-                .map(|x| x.clone())
-                .collect(),
-        ))
-    }
-
-    pub fn persist_unpersisted_stage(&self) -> Result<(), String> {
-        let events = self.events()?;
+    pub fn persist_stage(&self) -> Result<(), String> {
+        let old_snapshot = self.events()?.last_snapshot();
         let stage_mod_time = std::fs::metadata(&self.file)
             .unwrap()
             .modified()
@@ -175,49 +162,20 @@ impl File {
             .duration_since(std::time::UNIX_EPOCH)
             .unwrap()
             .as_secs();
-        let old_persisted: Vec<Delta> = events
-            .0
-            .iter()
-            .filter(|x| x.ts <= stage_mod_time)
-            .map(|x| x.clone())
-            .collect();
-        let old_events = Events(old_persisted);
-        eprintln!(
-            "stage_mod_time = {}, old events = {:?}",
-            stage_mod_time, old_events
-        );
-        let old_snapshot = old_events.snapshot()?;
-        eprintln!(
-            "self.persist_delta_at({:?}, {:?}",
-            old_snapshot,
-            self.stage().unwrap()
-        );
         self.persist_delta_at(old_snapshot, self.stage()?, stage_mod_time)
-        //self.stage_persisted()
     }
 
     pub fn stage_persisted(&self) -> Result<(), String> {
-        let persisted_as_snapshot = self.events_non_future()?.snapshot()?;
+        let persisted_as_snapshot = self.events()?.snapshot()?;
+        if persisted_as_snapshot != self.events()?.last_snapshot() {
+            self.append(Delta::snapshot(persisted_as_snapshot.clone()))?;
+        }
         let plaintext = serde_yaml::to_string(&persisted_as_snapshot).unwrap();
         let mut f = std::fs::File::create(&self.file).expect("failed to open file for writing");
         writeln!(f, "{}", plaintext).expect("failed to write");
         Ok(())
     }
 
-    pub fn persist_stage(&self) -> Result<(), String> {
-        self.persist_unpersisted_stage()?;
-
-        let persisted = self.events_non_future()?.snapshot()?;
-
-        let stage = self.stage()?;
-
-        eprintln!("persist delta...");
-        eprintln!(" persisted=before={:?}", &persisted);
-        eprintln!("     stage=aftere={:?}", &stage);
-        self.persist_delta(persisted, stage)?;
-        Ok(())
-    }
-
     pub fn persist_delta(&self, before: Vec<Task>, after: Vec<Task>) -> Result<(), String> {
         self.persist_delta_at(before, after, Delta::now_time())
     }
@@ -342,7 +300,7 @@ mod test_file {
             tests::file_contains(&d, "plain", "world");
 
             f.stage_persisted().unwrap();
-            assert_eq!(2, f.events().unwrap().0.len());
+            assert_eq!(3, f.events().unwrap().0.len());
             assert_eq!(2, f.stage().unwrap().len());
             tests::file_contains(&d, "plain", "- hello\n- world");
         });
@@ -358,6 +316,7 @@ mod test_file {
                 r#"
                     {"ts":1, "op":"Add", "task": "initial"}
                     {"ts":3, "op":"Add", "task": {"k":"v"}}
+                    {"ts":3, "op":"Snapshot", "task": null, "tasks": ["initial", 1, {"k":"v"}]}
                 "#,
             );
             tests::write_file(
@@ -370,19 +329,13 @@ mod test_file {
 
             let f = File::new(&d.path().join("plain").to_str().unwrap().to_string());
 
-            assert_eq!(3, f.events().unwrap().0.len());
+            assert_eq!(4, f.events().unwrap().0.len());
             assert_eq!(0, f.stage().unwrap().len());
             tests::file_contains(&d, "plain", "[]");
 
             f.persist_stage().unwrap();
-            assert_eq!(6, f.events().unwrap().0.len());
+            assert_eq!(7, f.events().unwrap().0.len());
             assert_eq!(0, f.stage().unwrap().len());
-            eprintln!("persist_stage | events | {:?}", f.events().unwrap().0);
-            eprintln!(
-                "persist_stage | events.snapshot | {:?}",
-                f.events().unwrap().snapshot()
-            );
-            eprintln!("persist_stage | stage | {:?}", f.stage().unwrap());
             tests::file_contains(&d, "plain", "[]");
 
             f.stage_persisted().unwrap();
@@ -393,7 +346,7 @@ mod test_file {
                 f.events().unwrap().snapshot().unwrap(),
             );
             assert_eq!(
-                6,
+                8,
                 f.events().unwrap().0.len(),
                 "{:?}",
                 f.events().unwrap().0
@@ -420,12 +373,13 @@ mod test_file {
                 ".plain.host_b",
                 r#"
                     {"ts":2, "op":"Add", "task": 1}
+                    {"ts":2, "op":"Snapshot", "task": null, "tasks": ["initial", 1]}
                 "#,
             );
 
             let f = File::new(&d.path().join("plain").to_str().unwrap().to_string());
 
-            assert_eq!(3, f.events().unwrap().0.len());
+            assert_eq!(4, f.events().unwrap().0.len());
             assert_eq!(2, f.stage().unwrap().len());
             tests::file_contains(&d, "plain", "- initial\n- 1");
 
@@ -435,14 +389,14 @@ mod test_file {
             tests::file_contains(&d, "plain", "- initial\n- 1");
 
             f.stage_persisted().unwrap();
-            assert_eq!(4, f.events().unwrap().0.len());
-            assert_eq!(2, f.stage().unwrap().len());
-            tests::file_contains(&d, "plain", "- initial\n- 1");
+            assert_eq!(5, f.events().unwrap().0.len());
+            assert_eq!(3, f.stage().unwrap().len());
+            tests::file_contains(&d, "plain", "- initial\n- 1\n- k: v");
         });
     }
 
     #[test]
-    fn test_persist_unpersisted_stage() {
+    fn test_persist_stage() {
         tests::with_dir(|d| {
             tests::write_file(&d, "plain", "- old\n- new");
             tests::write_file(
@@ -452,10 +406,9 @@ mod test_file {
                     r#"
                         {{"ts":1, "op":"Add", "task": "removed"}}
                         {{"ts":2, "op":"Add", "task": "old"}}
-                        {{"ts":{}, "op":"Add", "task": "was enqueued for persistence but obv removed"}}
-                        {{"ts":{}, "op":"Add", "task": "will be enqueued for persistence"}}
+                        {{"ts":2, "op":"Snapshot", "task": null, "tasks": ["removed", "old"]}}
+                        {{"ts":{}, "op":"Add", "task": "persisted but not snapshotted"}}
                     "#,
-                    Delta::now_time() - 50,
                     Delta::now_time() + 50,
                 )
                 .as_str(),
@@ -470,7 +423,7 @@ mod test_file {
 
             f.persist_stage().unwrap();
             assert_eq!(
-                7,
+                6,
                 f.events().unwrap().0.len(),
                 "events: {:?}",
                 f.events().unwrap()
@@ -485,9 +438,10 @@ mod test_file {
                 "{:?}",
                 f.events().unwrap().0
             );
-            assert_eq!(2, f.stage().unwrap().len(), "{:?}", f.stage().unwrap());
+            assert_eq!(3, f.stage().unwrap().len(), "{:?}", f.stage().unwrap());
             tests::file_contains(&d, "plain", "new");
             tests::file_contains(&d, "plain", "old");
+            tests::file_contains(&d, "plain", "persisted but not snapshotted");
         });
     }
 
@@ -549,9 +503,7 @@ mod test_file {
             );
             assert_eq!(0, f.stage().unwrap().len(), "{:?}", f.stage());
 
-            eprintln!("0 | {:?}", f.stage().unwrap());
             f.persist_stage().unwrap();
-            eprintln!("0 | {:?}", f.stage().unwrap());
             assert_eq!(
                 1,
                 f.events().unwrap().0.len(),
@@ -574,7 +526,7 @@ mod test_file {
 
             f.stage_persisted().unwrap();
             assert_eq!(
-                1,
+                2,
                 f.events().unwrap().0.len(),
                 "{:?}",
                 f.events().unwrap().0
@@ -609,6 +561,15 @@ impl Delta {
         }
     }
 
+    pub fn snapshot(tasks: Vec<Task>) -> Delta {
+        Delta {
+            ts: Self::now_time(),
+            op: Op::Snapshot,
+            task: Task(serde_yaml::Value::Null),
+            tasks: Some(tasks),
+        }
+    }
+
     pub fn add(task: Task) -> Delta {
         Self::add_at(task, Self::now_time())
     }
@@ -842,6 +803,16 @@ impl Events {
             .to_string()
     }
 
+    fn last_snapshot(&self) -> Vec<Task> {
+        for event in self.0.iter() {
+            match &event.op {
+                Op::Snapshot => return event.tasks.clone().unwrap(),
+                _ => {}
+            };
+        }
+        vec![]
+    }
+
     fn snapshot(&self) -> Result<Vec<Task>, String> {
         let mut result = vec![];
         for event in self.0.iter() {

pttodoest/src/testdata/.root.yaml.Bels-MacBook-Pro.local

@@ -1,4 +1,26 @@
-{"ts":1762884455,"patch":{"op":"replace","path":"","value":["read; https://topicpartition.io/blog/postgres-pubsub-queue-benchmarks","pglogical vs ha\n\n# api.git#breel/keys-620-pglogical-always-set-cr/2-user-survives-cr\n$ mise run pulsegres-new ^logical/toggl\n","drive; VERIFY spoc posts daily summary w/ unresolved","drive; VERIFY spoc refreshes summary w/ thread comment contianing 'refresh'","637; reconcile deploy if replicas wrong; https://github.com/renderinc/api/pull/26540/files","https://linear.app/render-com/issue/KEYS-633/add-3-when-max-connections-overridden-for-3-superuser-connections","https://linear.app/render-com/issue/KEYS-637/billing-resume-should-1-unsuspend-pg-in-cloudsql-2-unsuspend-pg-in-cr","https://linear.app/render-com/issue/KEYS-638/pgoperator-generates-new-ha-patroni-cert-every-reconcile-no-matter","pg; how2partition; https://renderinc.slack.com/archives/C0319NYCSSG/p1756357545556659?thread_ts=1756357467.613369&cid=C0319NYCSSG","pitr; backup purge cronjob for PL types","pg11 pgbackup doesnt write to envsetting mucked env key","incident io; teach spocbotvr to read slacks","userdb to internal; peer packages can use internal as userdb","fcr; cannot pitr because pgbackrest doesnt know wal spans thus pgexporter and friends cant know pitr works","etcd statefulset of 1 (for no random podname, no conflict, k8s ensures pod replace)\npatroni always\n","maher; https://slab.render.com/posts/hopes-and-dreams-blegf8fx#hdsyt-valkey-bundle","maher; shadow lizhi pm loops","maher; get more interviewers","maher; get concrete career and project plans so i can get promo in 2y; no manager to advocate","read; https://trychroma.com/engineering/wal3","read; https://github.com/renderinc/dashboard/pull/8883","read; https://litestream.io/getting-started/","kr\nto del gcloud old key\nie https://console.cloud.google.com/iam-admin/serviceaccounts/details/104206017956912104938/keys?hl=en&project=render-prod\n",{"subtasks":["","pitr\nhttps://slab.render.com/posts/pitr-as-a-service-health-abvnqx11\nmore aggressive alert autotune backup cores\nmore aggressive alert on MOAR backup cores\ncreate alert autotune archive-push cores\ncreate alert MOAR archive-push cores\n","cr; frontend","cr; cli.git","cr; public-api-schema.git; https://github.com/renderinc/public-api-schema/pull/407 STILL NEED EVENTS","cr; website.git","cr; changelog","ops; pgproxy rate limits 50ps 100burst; https://github.com/renderinc/dbproxy/pull/91","2873; no conn patroni if upgradeInProgressWithoutHA; https://github.com/renderinc/api/pull/26328","2733; only EnvSettings; https://github.com/renderinc/api/pull/25322/files","pg18; after cred rotation works, re enable e2e","655; pg18; pub api sch; https://github.com/renderinc/public-api-schema/pull/421","655; pg18; go generate pub api sch; https://github.com/renderinc/api/pull/26694","663; das; show status in /info; https://github.com/renderinc/dashboard/pull/9616","664; pg18; go gen terraform; https://github.com/renderinc/api/pull/26701","664; pg18; ga; push terraform.git#breel/keys-664-pg18","656; pg18; website; https://github.com/renderinc/website/pull/985/files","663; das; note disk cannot decrease even if autoscaled; https://github.com/renderinc/dashboard/pull/9621","pulsegres; pls let me keep my test emails; https://github.com/renderinc/api/pull/26741","pgup; restore view owner; https://github.com/renderinc/api/pull/26814","pgup; resync if missing resync; https://github.com/renderinc/api/pull/26817","pgup; replicas use $RESYNC; https://github.com/renderinc/api/pull/26878"],"todo":"blocked"}]}}
-{"ts":1762885026,"patch":{"op":"add","path":"/-","value":"hi"}}
-{"ts":1762915959,"patch":{"op":"add","path":"/-","value":"enqueued add"}}
-{"ts":1762915973,"patch":{"op":"remove","path":"/25"}}
+{"ts":1762915973,"op":"Add","task":"read; https://topicpartition.io/blog/postgres-pubsub-queue-benchmarks","tasks":null}
+{"ts":1762915973,"op":"Add","task":"pglogical vs ha\n\n# api.git#breel/keys-620-pglogical-always-set-cr/2-user-survives-cr\n$ mise run pulsegres-new ^logical/toggl\n","tasks":null}
+{"ts":1762915973,"op":"Add","task":"drive; VERIFY spoc posts daily summary w/ unresolved","tasks":null}
+{"ts":1762915973,"op":"Add","task":"drive; VERIFY spoc refreshes summary w/ thread comment contianing 'refresh'","tasks":null}
+{"ts":1762915973,"op":"Add","task":"637; reconcile deploy if replicas wrong; https://github.com/renderinc/api/pull/26540/files","tasks":null}
+{"ts":1762915973,"op":"Add","task":"https://linear.app/render-com/issue/KEYS-633/add-3-when-max-connections-overridden-for-3-superuser-connections","tasks":null}
+{"ts":1762915973,"op":"Add","task":"https://linear.app/render-com/issue/KEYS-637/billing-resume-should-1-unsuspend-pg-in-cloudsql-2-unsuspend-pg-in-cr","tasks":null}
+{"ts":1762915973,"op":"Add","task":"https://linear.app/render-com/issue/KEYS-638/pgoperator-generates-new-ha-patroni-cert-every-reconcile-no-matter","tasks":null}
+{"ts":1762915973,"op":"Add","task":"pg; how2partition; https://renderinc.slack.com/archives/C0319NYCSSG/p1756357545556659?thread_ts=1756357467.613369&cid=C0319NYCSSG","tasks":null}
+{"ts":1762915973,"op":"Add","task":"pitr; backup purge cronjob for PL types","tasks":null}
+{"ts":1762915973,"op":"Add","task":"pg11 pgbackup doesnt write to envsetting mucked env key","tasks":null}
+{"ts":1762915973,"op":"Add","task":"incident io; teach spocbotvr to read slacks","tasks":null}
+{"ts":1762915973,"op":"Add","task":"userdb to internal; peer packages can use internal as userdb","tasks":null}
+{"ts":1762915973,"op":"Add","task":"fcr; cannot pitr because pgbackrest doesnt know wal spans thus pgexporter and friends cant know pitr works","tasks":null}
+{"ts":1762915973,"op":"Add","task":"etcd statefulset of 1 (for no random podname, no conflict, k8s ensures pod replace)\npatroni always\n","tasks":null}
+{"ts":1762915973,"op":"Add","task":"maher; https://slab.render.com/posts/hopes-and-dreams-blegf8fx#hdsyt-valkey-bundle","tasks":null}
+{"ts":1762915973,"op":"Add","task":"maher; shadow lizhi pm loops","tasks":null}
+{"ts":1762915973,"op":"Add","task":"maher; get more interviewers","tasks":null}
+{"ts":1762915973,"op":"Add","task":"maher; get concrete career and project plans so i can get promo in 2y; no manager to advocate","tasks":null}
+{"ts":1762915973,"op":"Add","task":"read; https://trychroma.com/engineering/wal3","tasks":null}
+{"ts":1762915973,"op":"Add","task":"read; https://github.com/renderinc/dashboard/pull/8883","tasks":null}
+{"ts":1762915973,"op":"Add","task":"read; https://litestream.io/getting-started/","tasks":null}
+{"ts":1762915973,"op":"Add","task":"kr\nto del gcloud old key\nie https://console.cloud.google.com/iam-admin/serviceaccounts/details/104206017956912104938/keys?hl=en&project=render-prod\n","tasks":null}
+{"ts":1762915973,"op":"Add","task":{"subtasks":["","pitr\nhttps://slab.render.com/posts/pitr-as-a-service-health-abvnqx11\nmore aggressive alert autotune backup cores\nmore aggressive alert on MOAR backup cores\ncreate alert autotune archive-push cores\ncreate alert MOAR archive-push cores\n","cr; frontend","cr; cli.git","cr; public-api-schema.git; https://github.com/renderinc/public-api-schema/pull/407 STILL NEED EVENTS","cr; website.git","cr; changelog","ops; pgproxy rate limits 50ps 100burst; https://github.com/renderinc/dbproxy/pull/91","2873; no conn patroni if upgradeInProgressWithoutHA; https://github.com/renderinc/api/pull/26328","2733; only EnvSettings; https://github.com/renderinc/api/pull/25322/files","pg18; after cred rotation works, re enable e2e","655; pg18; pub api sch; https://github.com/renderinc/public-api-schema/pull/421","655; pg18; go generate pub api sch; https://github.com/renderinc/api/pull/26694","663; das; show status in /info; https://github.com/renderinc/dashboard/pull/9616","664; pg18; go gen terraform; https://github.com/renderinc/api/pull/26701","664; pg18; ga; push terraform.git#breel/keys-664-pg18","656; pg18; website; https://github.com/renderinc/website/pull/985/files","663; das; note disk cannot decrease even if autoscaled; https://github.com/renderinc/dashboard/pull/9621","pulsegres; pls let me keep my test emails; https://github.com/renderinc/api/pull/26741","pgup; restore view owner; https://github.com/renderinc/api/pull/26814","pgup; resync if missing resync; https://github.com/renderinc/api/pull/26817","pgup; replicas use $RESYNC; https://github.com/renderinc/api/pull/26878"],"todo":"blocked"},"tasks":null}
+{"ts":1762915973,"op":"Add","task":"hi","tasks":null}
+{"ts":1764635053,"op":"Snapshot","task":null,"tasks":["read; https://topicpartition.io/blog/postgres-pubsub-queue-benchmarks","pglogical vs ha\n\n# api.git#breel/keys-620-pglogical-always-set-cr/2-user-survives-cr\n$ mise run pulsegres-new ^logical/toggl\n","drive; VERIFY spoc posts daily summary w/ unresolved","drive; VERIFY spoc refreshes summary w/ thread comment contianing 'refresh'","637; reconcile deploy if replicas wrong; https://github.com/renderinc/api/pull/26540/files","https://linear.app/render-com/issue/KEYS-633/add-3-when-max-connections-overridden-for-3-superuser-connections","https://linear.app/render-com/issue/KEYS-637/billing-resume-should-1-unsuspend-pg-in-cloudsql-2-unsuspend-pg-in-cr","https://linear.app/render-com/issue/KEYS-638/pgoperator-generates-new-ha-patroni-cert-every-reconcile-no-matter","pg; how2partition; https://renderinc.slack.com/archives/C0319NYCSSG/p1756357545556659?thread_ts=1756357467.613369&cid=C0319NYCSSG","pitr; backup purge cronjob for PL types","pg11 pgbackup doesnt write to envsetting mucked env key","incident io; teach spocbotvr to read slacks","userdb to internal; peer packages can use internal as userdb","fcr; cannot pitr because pgbackrest doesnt know wal spans thus pgexporter and friends cant know pitr works","etcd statefulset of 1 (for no random podname, no conflict, k8s ensures pod replace)\npatroni always\n","maher; https://slab.render.com/posts/hopes-and-dreams-blegf8fx#hdsyt-valkey-bundle","maher; shadow lizhi pm loops","maher; get more interviewers","maher; get concrete career and project plans so i can get promo in 2y; no manager to advocate","read; https://trychroma.com/engineering/wal3","read; https://github.com/renderinc/dashboard/pull/8883","read; https://litestream.io/getting-started/","kr\nto del gcloud old key\nie https://console.cloud.google.com/iam-admin/serviceaccounts/details/104206017956912104938/keys?hl=en&project=render-prod\n",{"subtasks":["","pitr\nhttps://slab.render.com/posts/pitr-as-a-service-health-abvnqx11\nmore aggressive alert autotune backup cores\nmore aggressive alert on MOAR backup cores\ncreate alert autotune archive-push cores\ncreate alert MOAR archive-push cores\n","cr; frontend","cr; cli.git","cr; public-api-schema.git; https://github.com/renderinc/public-api-schema/pull/407 STILL NEED EVENTS","cr; website.git","cr; changelog","ops; pgproxy rate limits 50ps 100burst; https://github.com/renderinc/dbproxy/pull/91","2873; no conn patroni if upgradeInProgressWithoutHA; https://github.com/renderinc/api/pull/26328","2733; only EnvSettings; https://github.com/renderinc/api/pull/25322/files","pg18; after cred rotation works, re enable e2e","655; pg18; pub api sch; https://github.com/renderinc/public-api-schema/pull/421","655; pg18; go generate pub api sch; https://github.com/renderinc/api/pull/26694","663; das; show status in /info; https://github.com/renderinc/dashboard/pull/9616","664; pg18; go gen terraform; https://github.com/renderinc/api/pull/26701","664; pg18; ga; push terraform.git#breel/keys-664-pg18","656; pg18; website; https://github.com/renderinc/website/pull/985/files","663; das; note disk cannot decrease even if autoscaled; https://github.com/renderinc/dashboard/pull/9621","pulsegres; pls let me keep my test emails; https://github.com/renderinc/api/pull/26741","pgup; restore view owner; https://github.com/renderinc/api/pull/26814","pgup; resync if missing resync; https://github.com/renderinc/api/pull/26817","pgup; replicas use $RESYNC; https://github.com/renderinc/api/pull/26878"],"todo":"blocked"},"hi"]}