From 704e956dafe1283bae774456f5663af9ca82d7c1 Mon Sep 17 00:00:00 2001
From: bel <bel@bel.bel>
Date: Mon, 27 Jun 2022 18:34:31 -0600
Subject: [PATCH] cleanup

---
 scrape.sh | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/scrape.sh b/scrape.sh
index cc02bee..41c0972 100644
--- a/scrape.sh
+++ b/scrape.sh
@@ -1,6 +1,5 @@
 #! /bin/bash
 
-
 main() {
    ensure
 
@@ -114,6 +113,7 @@ should_scrape_book_chapter() {
    log foo:should_scrape_book_chapter $url
    if scrape_dndbeyond "$url" | grep -i "add to cart" &> /dev/null; then
       log "not purchased: $url"
+      _purge_cached_scrape_dndbeyond "$url"
       return 1
    fi
 }
@@ -154,9 +154,7 @@ list_chapters_in_book() {
 
 scrape_dndbeyond() {
    log foo:scrape_dndbeyond $@
-   local d="${TMPDIR}/scrape_cache"
-   mkdir -p "$d"
-   local f="$d/$(echo "$*" | base64)"
+   local f="$(_scrape_dndbeyond_cache_f "$@")"
    if [ -f "$f" ]; then
       if cat "$f" | grep .; then
          log "foo:scrape_dndbeyond: yes cached: $f: $@"
@@ -167,6 +165,19 @@ scrape_dndbeyond() {
    _scrape_dndbeyond "$@" | tee "$f"
 }
 
+_purge_cached_scrape_dndbeyond () {
+   local f="$(_scrape_dndbeyond_cache_f "$@")"
+   if [ -f "$f" ]; then
+      rm -f "$f"
+   fi
+}
+
+_scrape_dndbeyond_cache_f() {
+   local d="${TMPDIR}/scrape_cache"
+   mkdir -p "$d"
+   echo "$d/$(echo "$*" | base64 | tr -d '\n')"
+}
+
 _scrape_dndbeyond() {
    log foo:_scrape_dndbeyond $@
    rate_limit_1s scrape_dndbeyond
@@ -182,7 +193,7 @@ _scrape_dndbeyond() {
       -H 'sec-fetch-user: ?1' \
       -H 'sec-fetch-dest: document' \
       -H 'accept-language: en-US,en;q=0.9' \
-      -H 'Cookie: Preferences=undefined; Preferences=undefined; _pxhd=qvJb-g60h1HSymUFEiAmb-Noh35mNbUmcFgIE0qEiisCj/BqXrgrrsRUlODrjAaSGknpuHBn0S2b4rGTBFvaIA==:5/4beZX4ojNLKbAeZH9rQFcGIQ/ijS7lK29wvhLkgB4O0L2GFpRuU5FOpRztEiKg9EoUZr2cPB2Z8w3DKVkaKJPlwcIt23DM5PgQLZEPoA8=; ResponsiveSwitch.DesktopMode=1; LoginState=0ae9ef26-a1ae-4cb2-8af7-f18c21b99811; G_ENABLED_IDPS=google; CobaltSession=eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..FawE7bpRlluZsWq7Iny1xQ.nBX_qdZq5varuXNGkl70QeJ03JIRT3sxxQi6iCzazt-SjlK1hEuINX0sK85gJKVT.8wlUwv…sername=squeaky2x3; UserInfo={"UserId":109926924,"UserJoinDate":"2021-03-25","UserSessionId":"5ce909a9-34aa-4b15-9b6d-00ee48e13e05"}; AWSELB=17A593B6CA59C3C4856B812F84CD401A582EF08335AE71D0834BC2696233572C85266F7F9F4B5BF89DD4952FDA9AD462250417F8B7C4975F1633599CC9D18D74C3B57CBA; AWSELBCORS=17A593B6CA59C3C4856B812F84CD401A582EF08335AE71D0834BC2696233572C85266F7F9F4B5BF89DD4952FDA9AD462250417F8B7C4975F1633599CC9D18D74C3B57CBA; ddbSiteBanner:bb279d28-f265-49d5-ad41-6d716915bbdf=true; WarningNotification.Lock=1' \
+      -H 'Cookie: Preferences=undefined; Preferences=undefined; _pxhd=qvJb-g60h1HSymUFEiAmb-Noh35mNbUmcFgIE0qEiisCj/BqXrgrrsRUlODrjAaSGknpuHBn0S2b4rGTBFvaIA==:5/4beZX4ojNLKbAeZH9rQFcGIQ/ijS7lK29wvhLkgB4O0L2GFpRuU5FOpRztEiKg9EoUZr2cPB2Z8w3DKVkaKJPlwcIt23DM5PgQLZEPoA8=; ResponsiveSwitch.DesktopMode=1; LoginState=0ae9ef26-a1ae-4cb2-8af7-f18c21b99811; G_ENABLED_IDPS=google; CobaltSession=eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..FawE7bpRlluZsWq7Iny1xQ.nBX_qdZq5varuXNGkl70QeJ03JIRT3sxxQi6iCzazt-SjlK1hEuINX0sK85gJKVT.8wlUwv…sername=squeaky2x3; UserInfo={"UserId":109926924,"UserJoinDate":"2021-03-25","UserSessionId":"5ce909a9-34aa-4b15-9b6d-00ee48e13e05"}; ddbSiteBanner:bb279d28-f265-49d5-ad41-6d716915bbdf=true; AWSELB=17A593B6CA59C3C4856B812F84CD401A582EF083E14A2E133FB99235E1D3CC671549BC8DF3DD0A01DD4815C023F988F55E0E95275F44EAFDFB8393A8DEFA33DD0B90284D; AWSELBCORS=17A593B6CA59C3C4856B812F84CD401A582EF083E14A2E133FB99235E1D3CC671549BC8DF3DD0A01DD4815C023F988F55E0E95275F44EAFDFB8393A8DEFA33DD0B90284D; WarningNotification.Lock=1' \
       -H 'dnt: 1' \
       --compressed
    return