to meili splits into h2s

to_meili.sh

@@ -27,23 +27,45 @@ meili_queue() {
    curl "$MEILI"/indexes/scraped/updates
 }
 
+clean_id() {
+   echo "$*" | sed 's/[^a-zA-Z0-9_-]/_/g'
+}
+
 scraped_to_meili() {
    find "$SCRAPED" -type f \
    | sort \
    | while read -r md_path; do
-      id="$(echo "$md_path" | sed 's/[^a-zA-Z0-9_-]/_/g')"
+      file_id="$(clean_id "$md_path")"
-      content="$(cat "$md_path")"
+      h1="$(head -n 1 "$md_path" | sed 's/^# //')"
-      echo md_path=$md_path, content=${#content}
+      grep '^## ' "$md_path" | sed 's/^## //' | while read -r h2; do
-      curl -sS \
+         local content="$(
-         "$MEILI"/indexes/scraped/documents \
+            echo "# $h1"
-         -X POST \
+            echo
-         -d "$(
+            echo "## $h2"
-            goprintf '[{%q: %q, %q: %q}]' \
+            echo
-               "id" \
+            cat "$md_path" \
-               "$id" \
+            | sed -e '0,/^\#\# '"$h2"'/d' \
-               "content" \
+            | sed -e '/^\#\# .*/,$d' \
-               "$content"
+            | head -n 25
          )"
-      break
+         if [ $(echo "$content" | wc -l | awk '{print $NF}') -lt 5 ]; then
+            continue
+         fi
+         local id="$(clean_id "${file_id}_${h1}_${h2}")"
+         log id=$id
+         log h1=$h1
+         log h2=$h2
+         log content="${#content}"
+         curl -sS \
+            "$MEILI"/indexes/scraped/documents \
+            -X POST \
+            -d "$(
+               goprintf '[{%q: %q, %q: %q}]' \
+                  "id" \
+                  "$id" \
+                  "content" \
+                  "$content"
+            )"
+      done
    done
 }