shorted pid for scraped and do not change title

master
Bel LaPointe 2022-02-17 14:29:45 -07:00
parent 4dbe8072dd
commit 1bc0f17014
1 changed files with 3 additions and 0 deletions

View File

@ -95,6 +95,8 @@ crawl_with() {
local content="$(echo "$json" | jq -r .content)"
local crawlable_source="$(extract_crawlable_source "$content")"
notes put "$pid" "$(notes meta "$pid" | jq -r .Meta.Title)" "$crawlable_source"
local expanded=($($backend expand "$crawlable_source"))
log purge $crawlable_source:
@ -150,6 +152,7 @@ crawl_with() {
ID="${ID%/}"
if [ "${#expanded[@]}" -lt 2 ]; then
ID="$pid"
TITLE="$(notes meta "$ID" | jq -r .Meta.Title)"
CONTENT="$(printf "%s\n\n%s", "$crawlable_source" "$CONTENT")"
fi
log " $ID ($TITLE): ${#CONTENT}"