scrape_dndbeyond func

poc.sh

@@ -5,36 +5,41 @@ main() {
 }
 
 original() {
-   d=$(mktemp -d)
-   foo() {
-      curl -L -sS \
-         "${1:-https://www.dndbeyond.com/sources/phb}" \
-         --compressed \
-         -H 'Cookie: Preferences=undefined; Preferences=undefined; _pxhd=NHdkEjOXnAEHojER86i0Egmnm85GLvifiQxURPISwVtzWLNwuqQbIk1Y/2MjheQqNP4CsBXhuc4UG5-qcmLjYw==:hGx3TOEjeVaJPMfrXXnmyG7-btpQhZutwvpVBKwEbNXTAkEQvgmIs/faKB6s8537Nivj6epH9HpOWYURQeazeMwSMReW6EMazW09Rci-PJI=; ResponsiveSwitch.DesktopMode=1; ddbSiteBanner:00523ca3-81a4-4d2a-8f86-9e40273af2e2=true; LoginState=2c011278-8a3d-4cf7-a141-221f42694333; G_ENABLED_IDPS=google; CobaltSession=eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..vh9-_2G_aEjMvJ2PFAAE4w.eheZuOJLp-3NyiAJZzKqaSc91lH0ganCRsDZPYMGIG3265AppaHh0uMQLu_-P6WX.bHRi_gX8PKtQGZwVmNeb1Q; Preferences.Language=1; Preferences.TimeZoneID=1; ddbSiteBanner:d8879cae-c68e-4411-9ea0-a49d77fe5454=true; Ratings=null; _pxvid=f6abe72c-49a6-11ec-b844-57447254714c; ddbSiteBanner:fd981371-c501-4638-a64b-0d9a5c8da68b=true; Geo={%22region%22:%22ON%22%2C%22country%22:%22CA%22%2C%22continent%22:%22NA%22}; RequestVerificationToken=983cd9ce-c03c-4b8b-b6c1-a0561b39cb47; sublevel=ANON; Preferences=undefined; User.ID=109926924; User.Username=squeaky2x3; UserInfo={"UserId":109926924,"UserJoinDate":"2021-03-25","UserSessionId":"77896779-c672-42d1-903b-f6caba6ec599"}; pxcts=5eb1b230-525e-11ec-aa22-793bf1473016; WarningNotification.Lock=1' \
-         -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
-         -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \
-         -H 'Accept-Language: en-US,en;q=0.5' \
-         -H 'Referer: https://www.dndbeyond.com/' \
-         -H 'DNT: 1' \
-         -H 'Connection: keep-alive' \
-         -H 'Upgrade-Insecure-Requests: 1' \
-         -H 'Sec-Fetch-Dest: document' \
-         -H 'Sec-Fetch-Mode: navigate' \
-         -H 'Sec-Fetch-Site: same-origin' \
-         -H 'Sec-Fetch-User: ?1' \
-         -H 'Pragma: no-cache' \
-         -H 'Cache-Control: no-cache' \
-         -H 'TE: trailers';
-   }
-   for url in $(foo | grep -o 'https:..www.dndbeyond.com.compendium[^"]*' | sed 's/#.*//' | sort -u); do
+   for url in $(
+      scrape_dndbeyond https://www.dndbeyond.com/sources/phb \
+      | grep -o 'https:..www.dndbeyond.com.compendium[^"]*' \
+      | sed 's/#.*//' \
+      | sort -u
+   ); do
       echo $url
-      foo $url \
+      scrape_dndbeyond $url \
          | html_to_markdown \
-         | less || notes-server -root $d
+         | less
          break
       done
 }
 
+scrape_dndbeyond() {
+   curl -L -sS \
+      "${1:-https://www.dndbeyond.com/sources/phb}" \
+      --compressed \
+      -H 'Cookie: Preferences=undefined; Preferences=undefined; _pxhd=NHdkEjOXnAEHojER86i0Egmnm85GLvifiQxURPISwVtzWLNwuqQbIk1Y/2MjheQqNP4CsBXhuc4UG5-qcmLjYw==:hGx3TOEjeVaJPMfrXXnmyG7-btpQhZutwvpVBKwEbNXTAkEQvgmIs/faKB6s8537Nivj6epH9HpOWYURQeazeMwSMReW6EMazW09Rci-PJI=; ResponsiveSwitch.DesktopMode=1; ddbSiteBanner:00523ca3-81a4-4d2a-8f86-9e40273af2e2=true; LoginState=2c011278-8a3d-4cf7-a141-221f42694333; G_ENABLED_IDPS=google; CobaltSession=eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..vh9-_2G_aEjMvJ2PFAAE4w.eheZuOJLp-3NyiAJZzKqaSc91lH0ganCRsDZPYMGIG3265AppaHh0uMQLu_-P6WX.bHRi_gX8PKtQGZwVmNeb1Q; Preferences.Language=1; Preferences.TimeZoneID=1; ddbSiteBanner:d8879cae-c68e-4411-9ea0-a49d77fe5454=true; Ratings=null; _pxvid=f6abe72c-49a6-11ec-b844-57447254714c; ddbSiteBanner:fd981371-c501-4638-a64b-0d9a5c8da68b=true; Geo={%22region%22:%22ON%22%2C%22country%22:%22CA%22%2C%22continent%22:%22NA%22}; RequestVerificationToken=983cd9ce-c03c-4b8b-b6c1-a0561b39cb47; sublevel=ANON; Preferences=undefined; User.ID=109926924; User.Username=squeaky2x3; UserInfo={"UserId":109926924,"UserJoinDate":"2021-03-25","UserSessionId":"77896779-c672-42d1-903b-f6caba6ec599"}; pxcts=5eb1b230-525e-11ec-aa22-793bf1473016; WarningNotification.Lock=1' \
+      -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
+      -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \
+      -H 'Accept-Language: en-US,en;q=0.5' \
+      -H 'Referer: https://www.dndbeyond.com/' \
+      -H 'DNT: 1' \
+      -H 'Connection: keep-alive' \
+      -H 'Upgrade-Insecure-Requests: 1' \
+      -H 'Sec-Fetch-Dest: document' \
+      -H 'Sec-Fetch-Mode: navigate' \
+      -H 'Sec-Fetch-Site: same-origin' \
+      -H 'Sec-Fetch-User: ?1' \
+      -H 'Pragma: no-cache' \
+      -H 'Cache-Control: no-cache' \
+      -H 'TE: trailers';
+}
+
 html_to_markdown() {
    pandoc -f html -t markdown --ascii --toc --wrap=none --strip-comments -o - \
    | sed '/^:::.*/d' \