rate limit

poc.sh

@@ -1,10 +1,10 @@
 #! /bin/bash
 
 main() {
-   original "$@"
+   _main "$@"
 }
 
-original() {
+_main() {
    for url in $(
       scrape_dndbeyond https://www.dndbeyond.com/sources/phb \
       | grep -o 'https:..www.dndbeyond.com.compendium[^"]*' \
@@ -20,6 +20,7 @@ original() {
 }
 
 scrape_dndbeyond() {
+   rate_limit_1s scrape_dndbeyond
    curl -L -sS \
       "${1:-https://www.dndbeyond.com/sources/phb}" \
       --compressed \
@@ -51,6 +52,23 @@ html_to_markdown() {
    | sed "s/\\\\\([\"']\)/\1/g"
 }
 
+rate_limit_1s() {
+   local name="$1"
+   local d="${TMPDIR:-/tmp}"/rate_limited
+   mkdir -p "$d"
+   local last_run="$(date -r "$d/$name" +%s)"
+   local now="$(date +%s)"
+   local interval=2
+   echo "$(date +%s): should sleep while $(($(date +%s)-last_run)) < $interval" >&2
+   if [ -f "$d/$name" ]; then
+      while [ "$(($(date +%s)-last_run))" -lt $interval ]; do
+         sleep 1
+      done
+   fi
+   echo "$(date +%s): done sleeping" >&2
+   touch "$d/$name"
+}
+
 if [ "$0" == "$BASH_SOURCE" ]; then
    main "$@"
 fi