gogs
/
notea-de-me
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix jq doesnt like big keys without []

master
Bel LaPointe 2022-02-10 08:04:46 -07:00
parent 82ff38ba32
commit 829081ebed
1 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
app/crawler/main.sh
View File

@ -33,7 +33,8 @@ ids() {
crawl() { crawl() {
local cache_key="crawled $*" local cache_key="crawled $*"
if cache get "$cache_key"; then # TODO
if false && cache get "$cache_key"; then
return return
fi fi
_crawl "$@" | cache put "$cache_key" _crawl "$@" | cache put "$cache_key"
@ -76,18 +77,30 @@ crawl_with() {
local expanded=($($backend expand "$crawlable_source")) local expanded=($($backend expand "$crawlable_source"))
log expand $crawlable_source: log expand $crawlable_source:
for i in $(seq 1 $(("${#expanded[@]}"-1))); do one() {
export TITLE="$(echo "${expanded[i]}" | base64 --decode)" local i="$1"
export CONTENT="$($backend get "$crawlable_source" "${expanded[i]}")" export TITLE="$(
export ID="$(echo "$crawlable_source/$TITLE" | base64 | md5sum | awk '{print $1}')" echo "$i" | base64 --decode
export PID="$(echo $json | jq -r .id)" )"
export CONTENT="$(
$backend get "$crawlable_source" "$i"
)"
export ID="$(
echo "$crawlable_source/$TITLE" | base64 | md5sum | awk '{print $1}'
)"
export PID="$(
echo $json | jq -r .id
)"
log " $PID/$ID ($TITLE): ${#CONTENT}" log " $PID/$ID ($TITLE): ${#CONTENT}"
push_crawled push_crawled "$PID/$ID" "$TITLE" "$CONTENT"
}
for i in $(seq 1 $(("${#expanded[@]}"-1))); do
one "${expanded[i]}"
done done
} }
push_crawled() { push_crawled() {
notes put notes put "$@"
} }
is_crawlable() { is_crawlable() {