From b3d26a88c2cab7bde15f6e455ac401d729d3f2db Mon Sep 17 00:00:00 2001
From: Bel LaPointe <breel@qualtrics.com>
Date: Wed, 15 Dec 2021 15:35:30 -0700
Subject: [PATCH] meili scripting

---
 .gitignore          |  2 +-
 meilisearch.sh      |  6 ++++--
 poc.sh => scrape.sh |  0
 to_meili.sh         | 49 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 3 deletions(-)
 rename poc.sh => scrape.sh (100%)
 create mode 100644 to_meili.sh

diff --git a/.gitignore b/.gitignore
index b97de51..46e8bcc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
 **/*.sw*
 **/__pycache__
-/meilisearch
+/.meilisearch
 /scraped
 /data.ms
diff --git a/meilisearch.sh b/meilisearch.sh
index 12e91ab..4d7556d 100644
--- a/meilisearch.sh
+++ b/meilisearch.sh
@@ -1,10 +1,12 @@
 #! /bin/bash
 
-if [ ! -f ./meilisearch ]; then
+if [ ! -f ./.meilisearch ]; then
+   echo "https://docs.meilisearch.com/learn/getting_started/installation.html#configuration-options"
    curl -L https://install.meilisearch.com | sh
+   mv ./meilisearch ./.meilisearch
 fi
 args=("$@")
 if [ "${#args[@]}" == 0 ]; then
    args=(--db-path $PWD/data.ms --http-addr 0.0.0.0:7700)
 fi
-exec ./meilisearch "${args[@]}"
+exec ./.meilisearch "${args[@]}"
diff --git a/poc.sh b/scrape.sh
similarity index 100%
rename from poc.sh
rename to scrape.sh
diff --git a/to_meili.sh b/to_meili.sh
new file mode 100644
index 0000000..2a1deef
--- /dev/null
+++ b/to_meili.sh
@@ -0,0 +1,49 @@
+#! /bin/bash
+
+export MEILI="${MEILI:-"127.0.0.1:7700"}"
+export SCRAPED="${SCRAPED:-"./scraped"}"
+
+curl() {
+   $(which curl) -Ss -L -H 'Content-Type: application/json' "$@"
+   echo
+}
+
+log() {
+   echo "> $(date) > $*" >&2
+}
+
+meili_index() {
+   log index
+   curl "$MEILI"/indexes/scraped
+}
+
+meili_query() {
+   log query: "$*"
+   curl "$MEILI"/indexes/scraped/search -X POST -d "$(goprintf '{%q: %q}' "q" "$*")"
+}
+
+meili_queue() {
+   log queue
+   curl "$MEILI"/indexes/scraped/updates
+}
+
+scraped_to_meili() {
+   find "$SCRAPED" -type f \
+   | sort \
+   | while read -r md_path; do
+      id="$(echo "$md_path" | sed 's/[^a-zA-Z0-9_-]/_/g')"
+      content="$(cat "$md_path")"
+      echo md_path=$md_path, content=${#content}
+      curl -sS \
+         "$MEILI"/indexes/scraped/documents \
+         -X POST \
+         -d "$(
+            goprintf '[{%q: %q, %q: %q}]' \
+               "id" \
+               "$id" \
+               "content" \
+               "$content"
+         )"
+      break
+   done
+}