local-sandbox
/
dnd-beyond-scrape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some dumb grep problems

master
Bel LaPointe 2021-12-15 08:07:59 -07:00
parent b4335f764a
commit d682f836b4
1 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
poc.sh
View File

@ -4,11 +4,15 @@
main() { main() {
set -e set -e
set -o pipefail set -o pipefail
#set -x
local output="${1:-$(mktemp -d)}" local output="${1:-$(mktemp -d)}"
log "$output" log "$output"
#set -x
#scrape_book_chapter https://www.dndbeyond.com/compendium/rules/dmg/appendix-b-monster-lists
#scrape_book_chapter https://www.dndbeyond.com/compendium/rules/phb/credits | less #scrape_book_chapter https://www.dndbeyond.com/compendium/rules/phb/credits | less
#list_chapters_in_book https://www.dndbeyond.com/sources/sdw #list_chapters_in_book https://www.dndbeyond.com/sources/sdw
@ -47,7 +51,7 @@ scrape_book() {
local output="$1" local output="$1"
local book_url="$2" local book_url="$2"
if [ -d "$output" ] && ls "$output"/* && [ ! -f "$output/.wip" ]; then if [ -d "$output" ] && ls "$output"/* &> /dev/null && [ ! -f "$output/.wip" ]; then
log "$book_url already in $output" log "$book_url already in $output"
return return
fi fi
@ -129,11 +133,18 @@ html_to_markdown() {
| sed "s/\\\\\([\"']\)/\1/g" | sed "s/\\\\\([\"']\)/\1/g"
)" )"
local target="$( local target="$(
(echo "$out" | grep -q '^##* Appendix' && echo Appendix) \ if echo "$out" | grep '^##* Appendix' &> /dev/null; then
|| (echo "$out" | grep -q '^##* Chapter' && echo Chapter) \ echo Appendix
|| (echo "$out" | grep -q '^##* Introduction' && echo Introduction) \ elif echo "$out" | grep '^##* Chapter' &> /dev/null; then
|| (echo "$out" | grep -q '^##* Credits' && echo Credits) \ echo Chapter
|| echo Chapter elif echo "$out" | grep '^##* Introduction' &> /dev/null; then
echo Introduction
elif echo "$out" | grep '^##* Credits' &> /dev/null; then
echo Credits
else
log WARNING: NOTHING INDICITIVE OF TYPE OF CONTENT FOUND
echo Chapter
fi
)" )"
echo "$out" \ echo "$out" \
| sed -e '/^# '"$target"'/p' -e '0,/^# '"$target"'/d' | sed -e '/^# '"$target"'/p' -e '0,/^# '"$target"'/d'