package main import ( "bytes" "context" _ "embed" "encoding/base64" "encoding/json" "errors" "flag" "fmt" "io" "log" "net/http" "os" "os/exec" "os/signal" "path" "regexp" "strconv" "strings" "sync" "syscall" ) var ( Config struct { Port int Debug bool ChatBot struct { SessionD string PromptDelimiter string semaphore sync.Mutex WD string Command string N int } } //go:embed template.d/login.html htmlLogin []byte //go:embed template.d/index.html htmlIndex []byte //go:embed template.d/chatbot.html htmlChatBot []byte ) func main() { ctx, can := signal.NotifyContext(context.Background(), syscall.SIGINT) defer can() ctx, cleanup := contextWithCleanup(ctx) defer cleanup() config(ctx) log.Printf("%+v", Config) listenAndServe(ctx) log.Println("done") } func contextWithCleanup(ctx context.Context) (context.Context, func()) { m := map[int]func(){} ctx = context.WithValue(ctx, "__cleanup__", m) return ctx, func() { defer func() { recover() }() for _, v := range m { v() } } } func contextWithCleanupFunc(ctx context.Context, foo func()) { v := ctx.Value("__cleanup__") if v == nil { panic("cannot get context with cleanup func that doesnt have cleanup init") } m := v.(map[int]func()) m[len(m)] = foo } func config(ctx context.Context) { d, err := os.MkdirTemp(os.TempDir(), "ai.*") if err != nil { panic(err) } contextWithCleanupFunc(ctx, func() { os.RemoveAll(d) }) flag.IntVar(&Config.Port, "p", 37070, "port to listen on") flag.StringVar(&Config.ChatBot.SessionD, "chatbot-session-d", d, "dir to store chat bot sessions") flag.StringVar(&Config.ChatBot.PromptDelimiter, "chatbot-rp", "> ", "prompt delimiter prefixed by NAME/YOU") flag.StringVar(&Config.ChatBot.WD, "chatbot-working-d", "./llama.cpp", "working directory for chatbot") flag.StringVar(&Config.ChatBot.Command, "chatbot-cmd", "./main -m ./models/ggml-vic7b-uncensored-q5_1.bin --repeat_penalty 1.0", "chatbot cmd prefix") flag.IntVar(&Config.ChatBot.N, "chatbot-n", 256, "chatbot items to gen") flag.BoolVar(&Config.Debug, "debug", false, "debug mode") flag.Parse() } func listenAndServe(ctx context.Context) { wg := &sync.WaitGroup{} s := &http.Server{ Addr: fmt.Sprintf(":%d", Config.Port), Handler: http.HandlerFunc(handle), } wg.Add(1) go func() { defer wg.Done() if err := s.ListenAndServe(); err != nil && ctx.Err() == nil { panic(err) } }() <-ctx.Done() s.Close() wg.Wait() } func handle(w http.ResponseWriter, r *http.Request) { cookie, _ := ParseCookie(r) if err := _handle(w, r); err != nil { log.Printf("%s: %s %s: %v", cookie.MyName(), r.Method, r.URL.Path, err) } else { log.Printf("%s: %s %s", cookie.MyName(), r.Method, r.URL.Path) } } func _handle(w http.ResponseWriter, r *http.Request) error { //ctx, can := context.WithTimeout(r.Context(), time.Minute) //defer can() //r = r.WithContext(ctx) first := strings.Split(strings.TrimLeft(r.URL.Path, "/"), "/")[0] switch first { case "login": return handleLogin(w, r) case "api": return handleAPI(w, r) default: return handleUI(w, r) } } func handleLogin(w http.ResponseWriter, r *http.Request) error { switch r.Method { case http.MethodGet: w.Write(htmlLogin) return nil case http.MethodPost: err := r.ParseForm() if err != nil { return err } cookie, err := ParseCookie(r) if err != nil { return err } cookie.Inject(w) http.Redirect(w, r, "/", http.StatusTemporaryRedirect) return nil default: return handleNotFound(w, r) } } func handleUI(w http.ResponseWriter, r *http.Request) error { if _, err := ParseCookie(r); err != nil { http.Redirect(w, r, "/login", http.StatusTemporaryRedirect) return err } switch r.URL.Path { case "/": w.Write(htmlIndex) return nil case "/chatbot": if !Config.Debug { w.Write(htmlChatBot) } else { b, _ := os.ReadFile("./template.d/chatbot.html") w.Write(b) } return nil default: return handleNotFound(w, r) } } type Cookie struct { Name string } func ParseCookie(r *http.Request) (Cookie, error) { if r.URL.Path != "/login" { return parseCookieFromCookie(r) } cookie := Cookie{ Name: r.PostForm.Get("Name"), } return cookie, cookie.Verify() } func parseCookieFromCookie(r *http.Request) (Cookie, error) { cookie, err := r.Cookie("root") if err != nil { return Cookie{}, err } decoded, err := base64.URLEncoding.DecodeString(cookie.Value) if err != nil { return Cookie{}, err } var result Cookie if err := json.Unmarshal(decoded, &result); err != nil { return Cookie{}, err } return result, result.Verify() } func (cookie Cookie) MyName() string { return string(bytes.Join( regexp.MustCompile(`[a-zA-Z]`).FindAll([]byte(cookie.Name), -1), []byte(""), )) } func (cookie Cookie) Verify() error { if cookie.MyName() == "" { return fmt.Errorf("incomplete cookie") } return nil } func (cookie Cookie) Inject(w http.ResponseWriter) { c := &http.Cookie{ Name: "root", Value: cookie.Serialize(), } http.SetCookie(w, c) } func (cookie Cookie) Serialize() string { b, _ := json.Marshal(cookie) return base64.URLEncoding.EncodeToString(b) } func handleAPI(w http.ResponseWriter, r *http.Request) error { if _, err := ParseCookie(r); err != nil { http.Redirect(w, r, "/login", http.StatusTemporaryRedirect) return err } switch r.URL.Path { case "/api/v0/chatbot": return handleAPIChatBot(w, r) default: return handleNotFound(w, r) } } func handleNotFound(w http.ResponseWriter, r *http.Request) error { http.NotFound(w, r) return fmt.Errorf("not found: %s %s", r.Method, r.URL.Path) } func handleAPIChatBot(w http.ResponseWriter, r *http.Request) error { r.Header.Set("Content-Type", "application/x-www-form-urlencoded") err := r.ParseForm() if err != nil { return err } switch r.Method { case http.MethodPost: return handleAPIChatBotPost(w, r) case http.MethodPut: return handleAPIChatBotPut(w, r) case http.MethodGet: return handleAPIChatBotGet(w, r) default: return handleNotFound(w, r) } } func handleAPIChatBotGet(w http.ResponseWriter, r *http.Request) error { cookie, _ := ParseCookie(r) sessionD := path.Join(Config.ChatBot.SessionD, cookie.MyName()) f, err := os.Open(path.Join(sessionD, "prompt.txt")) if err != nil { return err } defer f.Close() io.Copy(w, f) return nil } func handleAPIChatBotPost(w http.ResponseWriter, r *http.Request) error { cookie, _ := ParseCookie(r) sessionD := path.Join(Config.ChatBot.SessionD, cookie.MyName()) if _, err := os.Stat(path.Join(sessionD)); err == nil { if err := os.RemoveAll(path.Join(sessionD)); err != nil { return err } } if err := os.MkdirAll(path.Join(sessionD, "chat.d"), os.ModePerm); err != nil { return err } prompt := r.PostForm.Get("Prompt") if len(prompt) == 0 { return errors.New("no prompt") } if err := os.WriteFile(path.Join(sessionD, "prompt.txt"), []byte(prompt), os.ModePerm); err != nil { return err } return handleAPIChatBotPut(w, r) } func handleAPIChatBotPut(w http.ResponseWriter, r *http.Request) error { message := r.PostForm.Get("Message") if len(message) == 0 { return errors.New("empty Message") } cookie, _ := ParseCookie(r) sessionD := path.Join(Config.ChatBot.SessionD, cookie.MyName()) promptF := path.Join(sessionD, "prompt.txt") inputF := path.Join(sessionD, "input.txt") cacheF := path.Join(sessionD, "cache.bin") reversePrompt := cookie.MyName() if len(reversePrompt) > 8 { reversePrompt = reversePrompt[:8] } reversePrompt = reversePrompt + Config.ChatBot.PromptDelimiter forwardPrompt := "YOU" + Config.ChatBot.PromptDelimiter if err := copyFile(inputF, promptF); err != nil { return err } if err := chatBotGenerateInitCacheF(r.Context(), cacheF, inputF); err != nil { return err } if err := appendFile(inputF, reversePrompt+message+"\n"+forwardPrompt); err != nil { return err } justNew, err := chatBotGenerateAndFillInputF(r.Context(), cacheF, inputF, reversePrompt, 0) if err != nil { return err } //log.Printf("generated for %s via %s: [%s]", cookie.MyName(), r.URL.Path, justNew) if err := os.Rename(inputF, promptF); err != nil { return err } w.Write(bytes.TrimSuffix(justNew, []byte(reversePrompt))) return nil } func chatBotGenerateInitCacheF(ctx context.Context, cacheF, inputF string) error { if _, err := os.Stat(cacheF); !os.IsNotExist(err) { return nil } commands := strings.Fields(Config.ChatBot.Command) commands = append(commands, "--batch-size", "8", "--prompt-cache", cacheF, "-f", inputF, "--n_predict", "1", ) command := exec.CommandContext( ctx, commands[0], commands[1:]..., ) command.Dir = Config.ChatBot.WD Config.ChatBot.semaphore.Lock() defer Config.ChatBot.semaphore.Unlock() if b, err := command.CombinedOutput(); err != nil { return fmt.Errorf("error generating cache with '%s': %w: %s", command.String(), err, b) } return nil } func chatBotGenerateAndFillInputF(ctx context.Context, cacheF, inputF, reversePrompt string, depth int) ([]byte, error) { if depth >= 3 { justNew := []byte("...\n" + reversePrompt) return justNew, _appendFile(inputF, string(justNew)) } commands := strings.Fields(Config.ChatBot.Command) commands = append(commands, "-f", inputF, "--prompt-cache-all", "--prompt-cache", cacheF, "-n", strconv.Itoa(Config.ChatBot.N), ) if len(reversePrompt) > 0 { commands = append(commands, "--reverse-prompt", reversePrompt, ) } command := exec.CommandContext( ctx, commands[0], commands[1:]..., ) command.Dir = Config.ChatBot.WD command.Stderr = io.Discard command.Stderr = log.Writer() stdout, err := command.StdoutPipe() if err != nil { return nil, err } buff := bytes.NewBuffer(nil) go func() { stdout.Read(make([]byte, 1)) //1 BOS byte io.Copy(buff, stdout) }() if err := func() error { Config.ChatBot.semaphore.Lock() defer Config.ChatBot.semaphore.Unlock() return command.Run() }(); err != nil { return nil, err } oldAndNew := buff.Bytes() priorContent, err := os.ReadFile(inputF) if err != nil { return nil, err } justNew := oldAndNew[len(priorContent):] if idx := bytes.Index(justNew, []byte(reversePrompt)); idx > 0 { justNew = justNew[:idx] } else if idx := bytes.LastIndex( append(priorContent, justNew...), []byte(reversePrompt), ); idx+len(reversePrompt) > len(priorContent) { justNew = justNew[:idx+len(reversePrompt)-len(priorContent)] } if err := _appendFile(inputF, string(justNew)); err != nil { return nil, err } if !bytes.HasSuffix(append(priorContent, justNew...), []byte(reversePrompt)) { more, err := chatBotGenerateAndFillInputF(ctx, cacheF, inputF, reversePrompt, depth+1) if err != nil { return nil, err } justNew = append(justNew, more...) } return justNew, nil } func copyFile(toF, fromF string) error { from, err := os.Open(fromF) if err != nil { return err } defer from.Close() to, err := os.Create(toF) if err != nil { return err } defer to.Close() io.Copy(to, from) return nil } func appendFile(toF, msg string) error { return _appendFile(toF, "\n"+msg+"\n") } func _appendFile(toF, msg string) error { f, err := os.OpenFile(toF, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return err } defer f.Close() if _, err := f.WriteString(msg); err != nil { return err } return nil }