From f4016da22007a951d4ff2adad0397a5daac42b35 Mon Sep 17 00:00:00 2001 From: bel Date: Sun, 15 Mar 2020 16:13:44 +0000 Subject: [PATCH] Fix job encode decode --- TODO.md | 23 ++++++++++-- config/config_test.go | 0 config/ns/ns.go | 6 ++++ logger/logger.go | 0 logger/logger_test.go | 0 main.go | 26 +++++++------- scheduler/errors.go | 0 scheduler/errors_test.go | 0 scheduler/job.go | 55 +++++++++++++++++++++-------- scheduler/job_test.go | 24 +++++++++---- scheduler/parser.go | 0 scheduler/parser_test.go | 0 scheduler/runner.go | 20 +++++++++++ scheduler/scheduler.go | 9 ++--- scheduler/scheduler_test.go | 0 server/routes.go | 58 +++++++++++++++++++++++++++++++ server/upserts.go | 46 ++++++++++++++++++++++++ testdata/5_jobs.cron | 5 +++ testdata/comment_only.cron | 0 testdata/hostname_per_5m.cron | 1 + testdata/hostname_per_second.cron | 0 21 files changed, 232 insertions(+), 41 deletions(-) mode change 100644 => 100755 TODO.md mode change 100644 => 100755 config/config_test.go create mode 100755 config/ns/ns.go mode change 100644 => 100755 logger/logger.go mode change 100644 => 100755 logger/logger_test.go mode change 100644 => 100755 scheduler/errors.go mode change 100644 => 100755 scheduler/errors_test.go mode change 100644 => 100755 scheduler/job.go mode change 100644 => 100755 scheduler/job_test.go mode change 100644 => 100755 scheduler/parser.go mode change 100644 => 100755 scheduler/parser_test.go mode change 100644 => 100755 scheduler/runner.go mode change 100644 => 100755 scheduler/scheduler.go mode change 100644 => 100755 scheduler/scheduler_test.go create mode 100755 server/upserts.go create mode 100755 testdata/5_jobs.cron mode change 100644 => 100755 testdata/comment_only.cron create mode 100755 testdata/hostname_per_5m.cron mode change 100644 => 100755 testdata/hostname_per_second.cron diff --git a/TODO.md b/TODO.md old mode 100644 new mode 100755 index de637ad..05a9451 --- a/TODO.md +++ b/TODO.md @@ -2,18 +2,22 @@ 1. UI to view 1. running job - 1. jobs + x jobs 1. job definition 1. next runtime 1. last runtime 1. last output + 1. add titles to jobs + 1. job title includes last pass/fail icon + 1. button to modify (copies to upsert form) + 1. button to delete 1. UI to mutate 1. submit job 1. delete job 1. pause jobs 1. interrupt job 1. JS - 1. ajax for json calls + x ajax for json calls # Backend @@ -21,5 +25,18 @@ x load from file 1. interrupt running jobs 1. temporarily disable jobs 1. json API + 1. list + 1. last run output + 1. last run pass/fail bool + 1. last run timestamp + 1. next run + 1. upsert + 1. delete job + 1. pause/disable job + 1. running job + 1. interrupt job 1. change cron to load the full job from storage so not holding big queued jobs in ram -1. add optional second for test main +x add optional second for test main +1. test main +1. add titles to jobs +1. namespace for jobs, output, lastrun, laststatus diff --git a/config/config_test.go b/config/config_test.go old mode 100644 new mode 100755 diff --git a/config/ns/ns.go b/config/ns/ns.go new file mode 100755 index 0000000..048ba27 --- /dev/null +++ b/config/ns/ns.go @@ -0,0 +1,6 @@ +package ns + +var ( + Jobs = []string{"jobs"} + Output = []string{"jobs", "output"} +) diff --git a/logger/logger.go b/logger/logger.go old mode 100644 new mode 100755 diff --git a/logger/logger_test.go b/logger/logger_test.go old mode 100644 new mode 100755 diff --git a/main.go b/main.go index f561675..e818697 100755 --- a/main.go +++ b/main.go @@ -3,8 +3,10 @@ package main import ( "local/firestormy/config" "local/firestormy/scheduler" + "local/firestormy/server" "local/lastn/lastn" "log" + "net/http" "os" "os/signal" "path/filepath" @@ -26,21 +28,19 @@ func main() { panic(err) } - /* - server := server.New() - if err := server.Routes(); err != nil { + server := server.New() + if err := server.Routes(); err != nil { + panic(err) + } + + //go EnqueueBackups() + + go func() { + log.Printf("Serving on %q", config.Port) + if err := http.ListenAndServe(config.Port, server); err != nil { panic(err) } - - go EnqueueBackups() - - go func() { - log.Printf("Serving on %q", config.Port) - if err := http.ListenAndServe(config.Port, server); err != nil { - panic(err) - } - }() - */ + }() // catch stop stop := make(chan os.Signal) diff --git a/scheduler/errors.go b/scheduler/errors.go old mode 100644 new mode 100755 diff --git a/scheduler/errors_test.go b/scheduler/errors_test.go old mode 100644 new mode 100755 diff --git a/scheduler/job.go b/scheduler/job.go old mode 100644 new mode 100755 index dc07d8c..4833d6d --- a/scheduler/job.go +++ b/scheduler/job.go @@ -4,18 +4,26 @@ import ( "bytes" "encoding/gob" "fmt" + "local/firestormy/config" + "local/firestormy/config/ns" "local/firestormy/logger" "os/exec" + "strings" + "time" "github.com/google/uuid" ) type Job struct { - Name string - Schedule string - Raw string - Runner Runner - foo func() + Name string + Schedule string + Raw string + Runner Runner + foo func() + LastStatus int + LastOutput string + LastRuntime time.Duration + LastRun time.Time } func NewJob(runner Runner, schedule, raw string) (*Job, error) { @@ -31,20 +39,33 @@ func newBashJob(schedule, sh string) (*Job, error) { if !validCron(schedule) { return nil, ErrBadCron } - return &Job{ + j := &Job{ Name: uuid.New().String(), Schedule: schedule, Raw: sh, Runner: Bash, - foo: func() { - cmd := exec.Command("bash", "-c", sh) - out, err := cmd.CombinedOutput() - if err != nil { - panic(err) - } - logger.New().Info(fmt.Sprintf("executed %s: %s", sh, out)) - }, - }, nil + } + j.foo = func() { + cmd := exec.Command("bash", "-c", sh) + j.LastRun = time.Now() + start := time.Now() + out, err := cmd.CombinedOutput() + j.LastRuntime = time.Since(start) + if err != nil { + out = []byte(fmt.Sprintf("error running command: %v: %v", err, out)) + } + j.LastOutput = strings.TrimSpace(string(out)) + if cmd != nil && cmd.ProcessState != nil { + j.LastStatus = cmd.ProcessState.ExitCode() + } + logger.New().Info(fmt.Sprintf("%+v", j)) + b, err := j.Encode() + if err == nil { + // TODO webpage doenst load post SET despite this returning nil + config.Store.Set(j.Name, b, ns.Jobs...) + } + } + return j, nil } func (j *Job) Run() { @@ -68,6 +89,10 @@ func (j *Job) Decode(b []byte) error { k, err := NewJob(j.Runner, j.Schedule, j.Raw) if err == nil { k.Name = j.Name + k.LastStatus = j.LastStatus + k.LastOutput = j.LastOutput + k.LastRuntime = j.LastRuntime + k.LastRun = j.LastRun *j = *k } return err diff --git a/scheduler/job_test.go b/scheduler/job_test.go old mode 100644 new mode 100755 index 63ce564..f4a4904 --- a/scheduler/job_test.go +++ b/scheduler/job_test.go @@ -3,9 +3,12 @@ package scheduler import ( "bytes" "io/ioutil" + "local/firestormy/config" "local/logb" + "local/storage" "os" "testing" + "time" ) func TestNewBashJobBadCron(t *testing.T) { @@ -16,6 +19,7 @@ func TestNewBashJobBadCron(t *testing.T) { } func TestNewBashJobAndRun(t *testing.T) { + config.Store = storage.NewMap() cases := []struct { sched string cmd string @@ -45,12 +49,7 @@ func TestNewBashJobAndRun(t *testing.T) { t.Error(err) continue } - func() { - defer func() { - recover() - }() - j.Run() - }() + j.Run() if !bytes.Contains(b.Bytes(), []byte(c.out)) { t.Errorf("(%s, %s) => %s", c.sched, c.cmd, b.Bytes()) } @@ -58,6 +57,7 @@ func TestNewBashJobAndRun(t *testing.T) { } func TestJobEncodeDecode(t *testing.T) { + config.Store = storage.NewMap() buff, clean := captureLog() defer clean() @@ -89,6 +89,18 @@ func TestJobEncodeDecode(t *testing.T) { if k.Name != j.Name { t.Error(k.Name, "vs", j.Name) } + if diff := k.LastRun.Unix() - j.LastRun.Unix(); (diff > 0 && diff < int64(time.Hour)) || (diff < 0 && -1*diff < int64(time.Hour)) { + t.Error(j.LastRun, "vs", k.LastRun) + } + if k.LastStatus != j.LastStatus { + t.Error(j.LastStatus, "vs", k.LastStatus) + } + if k.LastRuntime != j.LastRuntime { + t.Error(j.LastRuntime, "vs", k.LastRuntime) + } + if string(k.LastOutput) != string(j.LastOutput) { + t.Error(j.LastOutput, "vs", k.LastOutput) + } k.foo() if !bytes.Contains(buff.Bytes(), []byte(os.Getenv("HOSTNAME"))) { diff --git a/scheduler/parser.go b/scheduler/parser.go old mode 100644 new mode 100755 diff --git a/scheduler/parser_test.go b/scheduler/parser_test.go old mode 100644 new mode 100755 diff --git a/scheduler/runner.go b/scheduler/runner.go old mode 100644 new mode 100755 index cf5f083..3b93845 --- a/scheduler/runner.go +++ b/scheduler/runner.go @@ -5,3 +5,23 @@ type Runner int const ( Bash Runner = iota + 1 ) + +func (r Runner) String() string { + switch r { + case Bash: + return "bash" + default: + return "" + } +} + +func NewRunner(s string) Runner { + for _, r := range []Runner{ + Bash, + } { + if r.String() == s { + return r + } + } + return 0 +} diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go old mode 100644 new mode 100755 index 40f0d8a..f4897c5 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "local/firestormy/config" + "local/firestormy/config/ns" "local/firestormy/logger" "regexp" "strings" @@ -117,7 +118,7 @@ func (s *Scheduler) Stop() error { } func (s *Scheduler) List() ([]*Job, error) { - entries, err := config.Store.List(nil) + entries, err := config.Store.List(ns.Jobs) if err != nil { return nil, err } @@ -139,7 +140,7 @@ func (s *Scheduler) List() ([]*Job, error) { } func (s *Scheduler) loadJobFromStore(k string) (*Job, error) { - b, err := config.Store.Get(k) + b, err := config.Store.Get(k, ns.Jobs...) if err != nil { return nil, err } @@ -160,7 +161,7 @@ func (s *Scheduler) Add(j *Job) error { if err != nil { return err } - if err := config.Store.Set(j.Name, b); err != nil { + if err := config.Store.Set(j.Name, b, ns.Jobs...); err != nil { return err } s.running[j.Name] = entryID @@ -178,7 +179,7 @@ func (s *Scheduler) Remove(j *Job) error { if was == is { return ErrJobNotFound } - return config.Store.Set(j.Name, nil) + return config.Store.Set(j.Name, nil, ns.Jobs...) } func (s *Scheduler) getEntry(j *Job) (cron.EntryID, bool) { diff --git a/scheduler/scheduler_test.go b/scheduler/scheduler_test.go old mode 100644 new mode 100755 diff --git a/server/routes.go b/server/routes.go index c77cf1f..bca37c1 100755 --- a/server/routes.go +++ b/server/routes.go @@ -1,9 +1,15 @@ package server import ( + "encoding/json" "fmt" + "local/firestormy/config" + "local/firestormy/config/ns" + "local/firestormy/scheduler" "local/router" + "log" "net/http" + "sort" ) func (s *Server) Routes() error { @@ -12,6 +18,14 @@ func (s *Server) Routes() error { path string handler http.HandlerFunc }{ + { + path: fmt.Sprintf("/upserts"), + handler: s.gzip(s.authenticate(s.upserts)), + }, + { + path: fmt.Sprintf("/list"), + handler: s.gzip(s.authenticate(s.list)), + }, { path: fmt.Sprintf("%s%s", wildcard, wildcard), handler: s.gzip(s.authenticate(s.static)), @@ -29,3 +43,47 @@ func (s *Server) Routes() error { func (s *Server) static(w http.ResponseWriter, r *http.Request) { s.fileServer.ServeHTTP(w, r) } + +func (s *Server) upserts(w http.ResponseWriter, r *http.Request) { + upsert, err := newUpsertRequest(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + log.Println("received", upsert) + http.Error(w, "not impl", http.StatusNotImplemented) +} + +func (s *Server) list(w http.ResponseWriter, r *http.Request) { + jobs, err := config.Store.List(ns.Jobs) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + sort.Strings(jobs) + out := make([]map[string]interface{}, len(jobs)) + for i, job := range jobs { + out[i] = make(map[string]interface{}) + b, err := config.Store.Get(job, ns.Jobs...) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + j := &scheduler.Job{} + if err := j.Decode(b); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + out[i]["id"] = j.Name + out[i]["cron"] = j.Schedule + out[i]["language"] = j.Runner.String() + out[i]["script"] = j.Raw + out[i]["last"] = map[string]interface{}{ + "run": j.LastRun, + "runtime": j.LastRuntime, + "output": j.LastOutput, + "status": j.LastStatus, + } + } + json.NewEncoder(w).Encode(out) +} diff --git a/server/upserts.go b/server/upserts.go new file mode 100755 index 0000000..c3674a5 --- /dev/null +++ b/server/upserts.go @@ -0,0 +1,46 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "local/firestormy/config" + "local/firestormy/config/ns" + + "github.com/google/uuid" +) + +type upsertRequest struct { + ID string `json:"id"` + Language string `json:"language"` + Cron string `json:"cron"` + Script string `json:"script"` +} + +func newUpsertRequest(r io.Reader) (upsertRequest, error) { + u := upsertRequest{} + if err := json.NewDecoder(r).Decode(&u); err != nil { + return u, err + } + err := u.validate() + return u, err +} + +func (u *upsertRequest) validate() error { + if u.ID == "" { + u.ID = uuid.New().String() + } else if _, err := config.Store.Get(u.ID, ns.Jobs...); err != nil { + return fmt.Errorf("ID provided but not accessible: %v", err) + } + if u.Language == "" { + return errors.New("language required") + } + if u.Cron == "" { + return errors.New("cron required") + } + if u.Script == "" { + return errors.New("script required") + } + return nil +} diff --git a/testdata/5_jobs.cron b/testdata/5_jobs.cron new file mode 100755 index 0000000..fa0dfb3 --- /dev/null +++ b/testdata/5_jobs.cron @@ -0,0 +1,5 @@ +0 */5 * * * * echo first job +0 */15 * * * * echo second job +0 */25 * * * * echo third job +0 */35 * * * * echo fourth job +0 */45 * * * * true diff --git a/testdata/comment_only.cron b/testdata/comment_only.cron old mode 100644 new mode 100755 diff --git a/testdata/hostname_per_5m.cron b/testdata/hostname_per_5m.cron new file mode 100755 index 0000000..a9adfa2 --- /dev/null +++ b/testdata/hostname_per_5m.cron @@ -0,0 +1 @@ +0 */5 * * * * hostname diff --git a/testdata/hostname_per_second.cron b/testdata/hostname_per_second.cron old mode 100644 new mode 100755