PUT /api/v1/rpc/scrapeslack

main.go

@@ -64,12 +64,13 @@ func listenAndServe(ctx context.Context, cfg Config) chan error {
 func newHandler(cfg Config) http.HandlerFunc {
 	mux := http.NewServeMux()
 
-	mux.Handle("POST /api/v1/events/slack", http.HandlerFunc(newHandlerPostAPIV1EventsSlack(cfg)))
 	mux.Handle("GET /api/v1/eventnames", http.HandlerFunc(newHandlerGetAPIV1EventNames(cfg)))
 	mux.Handle("GET /api/v1/events", http.HandlerFunc(newHandlerGetAPIV1Events(cfg)))
 	mux.Handle("GET /api/v1/messages", http.HandlerFunc(newHandlerGetAPIV1Messages(cfg)))
 	mux.Handle("GET /api/v1/threads", http.HandlerFunc(newHandlerGetAPIV1Threads(cfg)))
 	mux.Handle("GET /api/v1/threads/{thread}", http.HandlerFunc(newHandlerGetAPIV1ThreadsThread(cfg)))
+	mux.Handle("POST /api/v1/events/slack", http.HandlerFunc(newHandlerPostAPIV1EventsSlack(cfg)))
+	mux.Handle("PUT /api/v1/rpc/scrapeslack", http.HandlerFunc(newHandlerPutAPIV1RPCScrapeSlack(cfg)))
 
 	return func(w http.ResponseWriter, r *http.Request) {
 		if cfg.Debug {
@@ -82,6 +83,61 @@ func newHandler(cfg Config) http.HandlerFunc {
 	}
 }
 
+func newHandlerPutAPIV1RPCScrapeSlack(cfg Config) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !basicAuth(cfg, w, r) {
+			return
+		}
+		channel := r.Header.Get("slack-channel")
+		token := r.Header.Get("slack-oauth-token")
+
+		req, err := http.NewRequest(http.MethodGet, "https://slack.com/api/conversations.history?channel="+channel, nil)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		req.Header.Set("Authorization", "Bearer "+token)
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+		defer resp.Body.Close()
+		defer io.Copy(io.Discard, resp.Body)
+
+		var page struct {
+			OK       bool
+			Messages []json.RawMessage
+		}
+		if err := json.NewDecoder(resp.Body).Decode(&page); err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		} else if !page.OK {
+			http.Error(w, "slack page was !.ok", http.StatusBadGateway)
+			return
+		}
+		errs := []error{}
+		for _, messageJSON := range page.Messages {
+			m, err := ParseSlack(messageJSON, cfg.AssetPattern, cfg.DatacenterPattern, cfg.EventNamePattern)
+			if errors.Is(err, ErrIrrelevantMessage) {
+			} else if err != nil {
+				errs = append(errs, err)
+			} else if err := cfg.storage.Upsert(r.Context(), m); err != nil {
+				errs = append(errs, err)
+			} else {
+				log.Printf("re-ingested %v from %s", m.ID, messageJSON)
+			}
+		}
+
+		if len(errs) > 0 {
+			http.Error(w, fmt.Sprint(errs), http.StatusInternalServerError)
+			return
+		}
+		json.NewEncoder(w).Encode(map[string]any{"scraped": len(page.Messages)})
+	}
+}
+
 func newHandlerGetAPIV1EventNames(cfg Config) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if !basicAuth(cfg, w, r) {