local-sandbox
/
dnd-beyond-scrape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix for credits, add wip for not redoing books

master
Bel LaPointe 2021-12-15 06:24:04 -07:00
parent d6e7061738
commit b4335f764a
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
poc.sh
View File

@ -6,9 +6,11 @@ main() {
set -o pipefail set -o pipefail
#set -x #set -x
local output="$(mktemp -d)" local output="${1:-$(mktemp -d)}"
log "$output" log "$output"
#scrape_book_chapter https://www.dndbeyond.com/compendium/rules/phb/credits | less
#list_chapters_in_book https://www.dndbeyond.com/sources/sdw #list_chapters_in_book https://www.dndbeyond.com/sources/sdw
#scrape_book_chapter https://www.dndbeyond.com/sources/phb/races | less #scrape_book_chapter https://www.dndbeyond.com/sources/phb/races | less
@ -20,7 +22,6 @@ main() {
book="${book%/}" book="${book%/}"
log book=$book log book=$book
scrape_book "$output/${book##*/}" "$book" scrape_book "$output/${book##*/}" "$book"
break
done done
@ -46,7 +47,13 @@ scrape_book() {
local output="$1" local output="$1"
local book_url="$2" local book_url="$2"
if [ -d "$output" ] && ls "$output"/* && [ ! -f "$output/.wip" ]; then
log "$book_url already in $output"
return
fi
mkdir -p "$output" mkdir -p "$output"
touch "$output/.wip"
local f="$(mktemp)" local f="$(mktemp)"
for chapter in $(list_chapters_in_book "$book_url"); do for chapter in $(list_chapters_in_book "$book_url"); do
@ -61,7 +68,10 @@ scrape_book() {
mv "$f" "$output"/"$title" mv "$f" "$output"/"$title"
done done
if [ -f "$f" ]; then
rm "$f" rm "$f"
fi
rm "$output/.wip"
} }
scrape_book_chapter() { scrape_book_chapter() {
@ -122,6 +132,7 @@ html_to_markdown() {
(echo "$out" | grep -q '^##* Appendix' && echo Appendix) \ (echo "$out" | grep -q '^##* Appendix' && echo Appendix) \
|| (echo "$out" | grep -q '^##* Chapter' && echo Chapter) \ || (echo "$out" | grep -q '^##* Chapter' && echo Chapter) \
|| (echo "$out" | grep -q '^##* Introduction' && echo Introduction) \ || (echo "$out" | grep -q '^##* Introduction' && echo Introduction) \
|| (echo "$out" | grep -q '^##* Credits' && echo Credits) \
|| echo Chapter || echo Chapter
)" )"
echo "$out" \ echo "$out" \