diff --git a/app/crawler/main.sh b/app/crawler/main.sh index 4d7f8af..14809fb 100644 --- a/app/crawler/main.sh +++ b/app/crawler/main.sh @@ -16,11 +16,11 @@ config() { export CACHE="${CACHE:-"$(mktemp -d)"}" mkdir -p "$CACHE" export CACHE_DURATION=$((60*50)) - export NOTEA_ADDR="${NOTEA_ADDR:-"http://localhost:3000"}" + export NOTES_ADDR="${NOTES_ADDR:-"http://localhost:3004"}" export GITLAB_PAT="$GITLAB_PAT" source ./gitlab.sh source ./cache.sh - source ./notea.sh + source ./notes.sh } log() { @@ -28,7 +28,7 @@ log() { } ids() { - notea ids + notes ids } crawl() { @@ -42,8 +42,12 @@ crawl() { _crawl() { log "crawling $*" local id="$1" - local json="$(notea get "$id")" - local content="$(echo "$json" | jq -r .content)" + local content="$(notes get "$id")" + local json="$( + printf '{"content": %s, "id": "%s"}' \ + "$(echo "$content" | jq -Rs)" \ + "$id" + )" if ! is_crawlable "$content"; then log "not crawlable: '${content:0:20}'..." return 0 @@ -83,7 +87,7 @@ crawl_with() { } push_crawled() { - notea put + notes put } is_crawlable() {