gogs
/
notea-de-me
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change main to notes

master
Bel LaPointe 2022-02-10 07:25:45 -07:00
parent e5e7276235
commit 642622cc02
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app/crawler/main.sh
View File

@ -16,11 +16,11 @@ config() {
export CACHE="${CACHE:-"$(mktemp -d)"}" export CACHE="${CACHE:-"$(mktemp -d)"}"
mkdir -p "$CACHE" mkdir -p "$CACHE"
export CACHE_DURATION=$((60*50)) export CACHE_DURATION=$((60*50))
export NOTEA_ADDR="${NOTEA_ADDR:-"http://localhost:3000"}" export NOTES_ADDR="${NOTES_ADDR:-"http://localhost:3004"}"
export GITLAB_PAT="$GITLAB_PAT" export GITLAB_PAT="$GITLAB_PAT"
source ./gitlab.sh source ./gitlab.sh
source ./cache.sh source ./cache.sh
source ./notea.sh source ./notes.sh
} }
log() { log() {
@ -28,7 +28,7 @@ log() {
} }
ids() { ids() {
notea ids notes ids
} }
crawl() { crawl() {
@ -42,8 +42,12 @@ crawl() {
_crawl() { _crawl() {
log "crawling $*" log "crawling $*"
local id="$1" local id="$1"
local json="$(notea get "$id")" local content="$(notes get "$id")"
local content="$(echo "$json" | jq -r .content)" local json="$(
printf '{"content": %s, "id": "%s"}' \
"$(echo "$content" | jq -Rs)" \
"$id"
)"
if ! is_crawlable "$content"; then if ! is_crawlable "$content"; then
log "not crawlable: '${content:0:20}'..." log "not crawlable: '${content:0:20}'..."
return 0 return 0
@ -83,7 +87,7 @@ crawl_with() {
} }
push_crawled() { push_crawled() {
notea put notes put
} }
is_crawlable() { is_crawlable() {