gogs
/
notea-de-me
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

main uses cache without special snowflake

master
Bel LaPointe 2022-02-01 10:59:11 -07:00
parent 451bef450c
commit c3a84c8e97
1 changed files with 7 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/crawler/main.sh
View File

@ -3,9 +3,7 @@
main() { main() {
config config
for id in $(ids); do for id in $(ids); do
if should_crawl "$id"; then
crawl "$id" crawl "$id"
fi
rewrite "$id" rewrite "$id"
done done
} }
@ -15,7 +13,6 @@ config() {
set -e set -e
export CACHE="${CACHE:-"$(mktemp -d)"}" export CACHE="${CACHE:-"$(mktemp -d)"}"
export CACHE_DURATION=$((60*5)) export CACHE_DURATION=$((60*5))
export CRAWL_INTERVAL=$((60*5))
export NOTEA_ADDR="${NOTEA_ADDR:-"http://localhost:3000"}" export NOTEA_ADDR="${NOTEA_ADDR:-"http://localhost:3000"}"
export GITLAB_PAT="$GITLAB_PAT" export GITLAB_PAT="$GITLAB_PAT"
source ./gitlab.sh source ./gitlab.sh
@ -29,20 +26,15 @@ ids() {
notea ids notea ids
} }
should_crawl() { crawl() {
local f="$CACHE/crawled.$1" local cache_key="crawled $*"
if [ ! -f "$f" ]; then if cache get "$cache_key"; then
return 0 return
fi fi
local last_crawled=$(date -r "$f" +%s) _crawl "$@" | cache put "$cache_key"
local now=$(date +%s)
if ((now-last_crawled < CRAWL_INTERVAL)); then
return 0
fi
return 1
} }
crawl() { _crawl() {
local id="$1" local id="$1"
local json="$(notea get "$id")" local json="$(notea get "$id")"
local content="$(echo "$json" | jq -r .content)" local content="$(echo "$json" | jq -r .content)"