shorted pid for scraped and do not change title
parent
4dbe8072dd
commit
1bc0f17014
|
|
@ -95,6 +95,8 @@ crawl_with() {
|
|||
local content="$(echo "$json" | jq -r .content)"
|
||||
local crawlable_source="$(extract_crawlable_source "$content")"
|
||||
|
||||
notes put "$pid" "$(notes meta "$pid" | jq -r .Meta.Title)" "$crawlable_source"
|
||||
|
||||
local expanded=($($backend expand "$crawlable_source"))
|
||||
|
||||
log purge $crawlable_source:
|
||||
|
|
@ -150,6 +152,7 @@ crawl_with() {
|
|||
ID="${ID%/}"
|
||||
if [ "${#expanded[@]}" -lt 2 ]; then
|
||||
ID="$pid"
|
||||
TITLE="$(notes meta "$ID" | jq -r .Meta.Title)"
|
||||
CONTENT="$(printf "%s\n\n%s", "$crawlable_source" "$CONTENT")"
|
||||
fi
|
||||
log " $ID ($TITLE): ${#CONTENT}"
|
||||
|
|
|
|||
Loading…
Reference in New Issue