asses add modtime column to skip cksumming for dupe work

src/asses/db.go

@@ -36,6 +36,7 @@ func Record(ctx context.Context) error {
 type last struct {
 	T        time.Time `json:"checked_at"`
 	Cksum    string    `json:"cksum"`
+	Modified time.Time `json:"modified"`
 }
 
 func checkLast(ctx context.Context, p string) (last, error) {
@@ -44,26 +45,26 @@ func checkLast(ctx context.Context, p string) (last, error) {
 	}
 
 	return db.QueryOne[last](ctx, `
-		SELECT checked_at, cksum
+		SELECT checked_at, cksum, modified
 		FROM "asses.checks"
 		WHERE p=$1
 	`, p)
 }
 
-func checked(ctx context.Context, p, cksum string) error {
+func checked(ctx context.Context, p, cksum string, modified time.Time) error {
 	if err := initDB(ctx); err != nil {
 		return err
 	}
 
 	return db.Exec(ctx, `
 		INSERT INTO "asses.checks"
-		(p, checked_at, cksum)
+		(p, checked_at, cksum, modified)
-		VALUES ($1, $2, $3)
+		VALUES ($1, $2, $3, $4)
 
 		ON CONFLICT DO UPDATE
-		SET checked_at=$2, cksum=$3
+		SET checked_at=$2, cksum=$3, modified=$4
 		WHERE p=$1
-	`, p, time.Now(), cksum)
+	`, p, time.Now(), cksum, modified)
 }
 
 func initDB(ctx context.Context) error {
@@ -77,5 +78,6 @@ func initDB(ctx context.Context) error {
 			checked_at TIMESTAMP NOT NULL,
 			cksum TEXT NOT NULL
 		)`,
+		`ALTER TABLE "asses.checks" ADD COLUMN "modified" TIMESTAMP`,
 	})
 }

src/asses/db_test.go

@@ -2,8 +2,10 @@ package asses
 
 import (
 	"context"
+	"math"
 	"show-rss/src/db"
 	"testing"
+	"time"
 )
 
 func TestLast(t *testing.T) {
@@ -15,9 +17,10 @@ func TestLast(t *testing.T) {
 		t.Fatal(last)
 	}
 
-	if err := checked(ctx, "p", "cksum"); err != nil {
+	modtime := time.Now().Add(-5 * time.Minute)
+	if err := checked(ctx, "p", "cksum", modtime); err != nil {
 		t.Fatal(err)
-	} else if err := checked(ctx, "p", "cksum"); err != nil {
+	} else if err := checked(ctx, "p", "cksum", modtime); err != nil {
 		t.Fatal(err)
 	}
 
@@ -25,5 +28,7 @@ func TestLast(t *testing.T) {
 		t.Fatal(err)
 	} else if last.T.IsZero() || last.Cksum != "cksum" {
 		t.Fatal(last)
+	} else if math.Abs(float64(last.Modified.Sub(modtime))) > float64(time.Second) {
+		t.Fatalf("modified not uploaded: %v vs. %v (diff of %v)", last.Modified, modtime, last.Modified.Sub(modtime))
 	}
 }

src/asses/one.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
+	"fmt"
 	"io"
 	"log"
 	"math/rand"
@@ -13,34 +14,46 @@ import (
 )
 
 func One(ctx context.Context, p string) error {
-	shortp := path.Join("...", path.Dir(p), path.Base(p))
+	shortp := path.Join("...", path.Base(path.Dir(p)), path.Base(p))
 
 	last, err := checkLast(ctx, p)
 	if err != nil {
 		return err
 	}
 
-	if last.T.IsZero() {
+	if threshold := 20 + time.Duration(rand.Int()%10)*24*time.Hour; time.Since(last.T) > threshold {
-		log.Printf("asses.One(%s) // never seen before", shortp)
+	} else if stat, err := os.Stat(p); err != nil {
-	} else if cksum, err := Cksum(p); err != nil {
+		return fmt.Errorf("cannot stat %s: %w", p, err)
-		return err
+	} else if stat.ModTime() == last.Modified {
-	} else if cksum != last.Cksum {
+		log.Printf("asses.One(%s) // unmodified since %v", shortp, last.T)
-		log.Printf("asses.One(%s) // cksum changed", shortp)
-	} else if threshold := 20 + time.Duration(rand.Int()%10)*24*time.Hour; time.Since(last.T) < threshold {
-		log.Printf("asses.One(%s) // lt %v since last review", shortp, threshold.Hours())
 		return nil
 	}
 
+	if err := func() error {
+		if cksum, err := Cksum(p); err != nil {
+			return err
+		} else if cksum == last.Cksum {
+			log.Printf("asses.One(%s) // cksum unchanged since %v", shortp, last.T)
+		} else {
 			log.Printf("asses.deport(%s)...", shortp)
 			if err := deport(ctx, p); err != nil {
 				return err
 			}
+		}
+		return nil
+	}(); err != nil {
+		return err
+	}
 
 	cksum, err := Cksum(p)
 	if err != nil {
 		return err
 	}
-	return checked(ctx, p, cksum)
+	stat, err := os.Stat(p)
+	if err != nil {
+		return err
+	}
+	return checked(ctx, p, cksum, stat.ModTime())
 }
 
 func Cksum(p string) (string, error) {