diff --git a/poc.sh b/poc.sh index 2dd3481..03e9851 100644 --- a/poc.sh +++ b/poc.sh @@ -1,10 +1,10 @@ #! /bin/bash main() { - original "$@" + _main "$@" } -original() { +_main() { for url in $( scrape_dndbeyond https://www.dndbeyond.com/sources/phb \ | grep -o 'https:..www.dndbeyond.com.compendium[^"]*' \ @@ -20,6 +20,7 @@ original() { } scrape_dndbeyond() { + rate_limit_1s scrape_dndbeyond curl -L -sS \ "${1:-https://www.dndbeyond.com/sources/phb}" \ --compressed \ @@ -51,6 +52,23 @@ html_to_markdown() { | sed "s/\\\\\([\"']\)/\1/g" } +rate_limit_1s() { + local name="$1" + local d="${TMPDIR:-/tmp}"/rate_limited + mkdir -p "$d" + local last_run="$(date -r "$d/$name" +%s)" + local now="$(date +%s)" + local interval=2 + echo "$(date +%s): should sleep while $(($(date +%s)-last_run)) < $interval" >&2 + if [ -f "$d/$name" ]; then + while [ "$(($(date +%s)-last_run))" -lt $interval ]; do + sleep 1 + done + fi + echo "$(date +%s): done sleeping" >&2 + touch "$d/$name" +} + if [ "$0" == "$BASH_SOURCE" ]; then main "$@" fi