From 9e1b252af2891663d019c8e808bba4f12d7a82c9 Mon Sep 17 00:00:00 2001
From: bel <bel@bel.bel>
Date: Sat, 17 Jun 2023 10:37:05 -0600
Subject: [PATCH] ok except caching means segfault so tahts weird

---
 vicuna-tools.d/main.go | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/vicuna-tools.d/main.go b/vicuna-tools.d/main.go
index 004b23d..a5b00af 100644
--- a/vicuna-tools.d/main.go
+++ b/vicuna-tools.d/main.go
@@ -295,19 +295,48 @@ func handleAPIChatBotPut(w http.ResponseWriter, r *http.Request) error {
 	promptF := path.Join(sessionD, "prompt.txt")
 	inputF := path.Join(sessionD, "input.txt")
 	cacheF := path.Join(sessionD, "cache.bin")
+	reversePrompt := cookie.Name + "///"
 	if err := copyFile(inputF, promptF); err != nil {
 		return err
 	}
-	if err := appendFile(inputF, message); err != nil {
+	if err := appendFile(inputF, reversePrompt+message); err != nil {
 		return err
 	}
+	if _, err := os.Stat(cacheF); false && os.IsNotExist(err) {
+		if err := func() error {
+			commands := strings.Fields(Config.ChatBot.Command)
+			commands = append(commands,
+				"--batch-size", "8",
+				"--prompt-cache", cacheF,
+				"-f", inputF,
+				"--n_predict", "1",
+			)
+			command := exec.CommandContext(
+				r.Context(),
+				commands[0],
+				commands[1:]...,
+			)
+			command.Dir = Config.ChatBot.WD
+
+			Config.ChatBot.semaphore.Lock()
+			defer Config.ChatBot.semaphore.Unlock()
+
+			if b, err := command.CombinedOutput(); err != nil {
+				return fmt.Errorf("error generating cache: %w: %s", err, b)
+			}
+			return nil
+		}(); err != nil {
+			return err
+		}
+	}
 
 	commands := strings.Fields(Config.ChatBot.Command)
 	commands = append(commands,
 		"-f", inputF,
-		"--prompt-cache-all",
-		"--prompt-cache", cacheF,
+		//"--prompt-cache-all",
+		//"--prompt-cache", cacheF,
 		"-n", strconv.Itoa(Config.ChatBot.N),
+		"--reverse-prompt", reversePrompt,
 	)
 	command := exec.CommandContext(
 		r.Context(),
@@ -323,6 +352,7 @@ func handleAPIChatBotPut(w http.ResponseWriter, r *http.Request) error {
 	}
 	buff := bytes.NewBuffer(nil)
 	go func() {
+		stdout.Read(make([]byte, 1))
 		io.Copy(buff, stdout)
 	}()