From 1bc0f170145d19c214a1d7adb3da060aa14c99ea Mon Sep 17 00:00:00 2001
From: Bel LaPointe <breel@qualtrics.com>
Date: Thu, 17 Feb 2022 14:29:45 -0700
Subject: [PATCH] shorted pid for scraped and do not change title

---
 crawler/main.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/crawler/main.sh b/crawler/main.sh
index 95eadc4..6beaec8 100644
--- a/crawler/main.sh
+++ b/crawler/main.sh
@@ -95,6 +95,8 @@ crawl_with() {
 	local content="$(echo "$json" | jq -r .content)"
 	local crawlable_source="$(extract_crawlable_source "$content")"
 
+	notes put "$pid" "$(notes meta "$pid" | jq -r .Meta.Title)" "$crawlable_source"
+
 	local expanded=($($backend expand "$crawlable_source"))
 
 	log purge $crawlable_source:
@@ -150,6 +152,7 @@ crawl_with() {
 		ID="${ID%/}"
 		if [ "${#expanded[@]}" -lt 2 ]; then
 			ID="$pid"
+			TITLE="$(notes meta "$ID" | jq -r .Meta.Title)"
 			CONTENT="$(printf "%s\n\n%s", "$crawlable_source" "$CONTENT")"
 		fi
 		log "	$ID ($TITLE): ${#CONTENT}"