gogs
/
notea-de-me
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

shorted pid for scraped and do not change title

master
Bel LaPointe 2022-02-17 14:29:45 -07:00
parent 4dbe8072dd
commit 1bc0f17014
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawler/main.sh
View File

@ -95,6 +95,8 @@ crawl_with() {
local content="$(echo "$json" | jq -r .content)" local content="$(echo "$json" | jq -r .content)"
local crawlable_source="$(extract_crawlable_source "$content")" local crawlable_source="$(extract_crawlable_source "$content")"
notes put "$pid" "$(notes meta "$pid" | jq -r .Meta.Title)" "$crawlable_source"
local expanded=($($backend expand "$crawlable_source")) local expanded=($($backend expand "$crawlable_source"))
log purge $crawlable_source: log purge $crawlable_source:
@ -150,6 +152,7 @@ crawl_with() {
ID="${ID%/}" ID="${ID%/}"
if [ "${#expanded[@]}" -lt 2 ]; then if [ "${#expanded[@]}" -lt 2 ]; then
ID="$pid" ID="$pid"
TITLE="$(notes meta "$ID" | jq -r .Meta.Title)"
CONTENT="$(printf "%s\n\n%s", "$crawlable_source" "$CONTENT")" CONTENT="$(printf "%s\n\n%s", "$crawlable_source" "$CONTENT")"
fi fi
log " $ID ($TITLE): ${#CONTENT}" log " $ID ($TITLE): ${#CONTENT}"